


'''simple docstring'''
import warnings
from contextlib import contextmanager

from ...processing_utils import ProcessorMixin
class      lowercase			(							A__      ):

		"""simple docstring"""

		_a									=       'Speech2TextFeatureExtractor'
		_a									=       'Speech2TextTokenizer'




		def __init__(	self					,     UpperCamelCase_					,     UpperCamelCase_							):

							'''simple docstring'''


							super().__init__(UpperCamelCase_					,     UpperCamelCase_							)
							UpperCamelCase__     :List[str]							      =				self.feature_extractor
							UpperCamelCase__     :Optional[Any]							      =				False




		def __call__(	self					,     *UpperCamelCase_					,     **UpperCamelCase_							):

							'''simple docstring'''


							if self._in_target_context_manager:
												return self.current_processor(*UpperCamelCase_					,     **UpperCamelCase_							)

							if "raw_speech" in kwargs:
												warnings.warn('''Using `raw_speech` as a keyword argument is deprecated. Use `audio` instead.'''							)
												UpperCamelCase__     :Optional[Any]							      =				kwargs.pop('''raw_speech'''							)
							else:
												UpperCamelCase__     :List[Any]							      =				kwargs.pop('''audio'''					,     UpperCamelCase_							)
							UpperCamelCase__     :List[Any]							      =				kwargs.pop('''sampling_rate'''					,     UpperCamelCase_							)
							UpperCamelCase__     :Dict							      =				kwargs.pop('''text'''					,     UpperCamelCase_							)
							if len(UpperCamelCase_							) > 0:
												UpperCamelCase__     :Tuple							      =				args[0]
												UpperCamelCase__     :Optional[int]							      =				args[1:]

							if audio is None and text is None:
												raise ValueError('''You need to specify either an `audio` or `text` input to process.'''							)

							if audio is not None:
												UpperCamelCase__     :Any							      =				self.feature_extractor(UpperCamelCase_					,     *UpperCamelCase_					,     sampling_rate=UpperCamelCase_					,     **UpperCamelCase_							)
							if text is not None:
												UpperCamelCase__     :Dict							      =				self.tokenizer(UpperCamelCase_					,     **UpperCamelCase_							)

							if text is None:
												return inputs
							elif audio is None:
												return encodings
							else:
												UpperCamelCase__     :Dict							      =				encodings['''input_ids''']
												return inputs




		def 					lowerCAmelCase__       (	self					,     *UpperCamelCase_					,     **UpperCamelCase_							):

							'''simple docstring'''


							return self.tokenizer.batch_decode(*UpperCamelCase_					,     **UpperCamelCase_							)




		def 					lowerCAmelCase__       (	self					,     *UpperCamelCase_					,     **UpperCamelCase_							):

							'''simple docstring'''


							return self.tokenizer.decode(*UpperCamelCase_					,     **UpperCamelCase_							)




		@contextmanager
		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							warnings.warn(
							    '''`as_target_processor` is deprecated and will be removed in v5 of Transformers. You can process your '''
							    '''labels by using the argument `text` of the regular `__call__` method (either in the same call as '''
							    '''your audio inputs, or in a separate call.'''							)
							UpperCamelCase__     :Optional[Any]							      =				True
							UpperCamelCase__     :Any							      =				self.tokenizer
							yield
							UpperCamelCase__     :Any							      =				self.feature_extractor
							UpperCamelCase__     :Any							      =				False



'''simple docstring'''
import json
import os
import unittest

from transformers.models.roc_bert.tokenization_roc_bert import (
    VOCAB_FILES_NAMES,
    RoCBertBasicTokenizer,
    RoCBertTokenizer,
    RoCBertWordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english
@require_tokenizers
class      lowercase			(							A__			,	unittest.TestCase      ):

		"""simple docstring"""

		_a									=       RoCBertTokenizer
		_a									=       None
		_a									=       False
		_a									=       True
		_a									=       filter_non_english




		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							super().setUp()

							UpperCamelCase__     :List[str]							      =				['''[UNK]''', '''[CLS]''', '''[SEP]''', '''[PAD]''', '''[MASK]''', '''你''', '''好''', '''是''', '''谁''', '''a''', '''b''', '''c''', '''d''']
							UpperCamelCase__     :Optional[Any]							      =				{}
							UpperCamelCase__     :str							      =				{}
							for i, value in enumerate(UpperCamelCase_							):
												UpperCamelCase__     :List[Any]							      =				i
												UpperCamelCase__     :List[str]							      =				i
							UpperCamelCase__     :Any							      =				os.path.join(self.tmpdirname					,     VOCAB_FILES_NAMES['''vocab_file''']							)
							UpperCamelCase__     :Optional[Any]							      =				os.path.join(self.tmpdirname					,     VOCAB_FILES_NAMES['''word_shape_file''']							)
							UpperCamelCase__     :List[str]							      =				os.path.join(self.tmpdirname					,     VOCAB_FILES_NAMES['''word_pronunciation_file''']							)
							with open(self.vocab_file					,     '''w'''					,     encoding='''utf-8'''							) as vocab_writer:
												vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]							)							)
							with open(self.word_shape_file					,     '''w'''					,     encoding='''utf-8'''							) as word_shape_writer:
												json.dump(UpperCamelCase_					,     UpperCamelCase_					,     ensure_ascii=UpperCamelCase_							)
							with open(self.word_pronunciation_file					,     '''w'''					,     encoding='''utf-8'''							) as word_pronunciation_writer:
												json.dump(UpperCamelCase_					,     UpperCamelCase_					,     ensure_ascii=UpperCamelCase_							)




		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							UpperCamelCase__     :Tuple							      =				self.tokenizer_class(self.vocab_file					,     self.word_shape_file					,     self.word_pronunciation_file							)

							UpperCamelCase__     :Optional[int]							      =				tokenizer.tokenize('''你好[SEP]你是谁'''							)
							self.assertListEqual(UpperCamelCase_					,     ['''你''', '''好''', '''[SEP]''', '''你''', '''是''', '''谁''']							)
							self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCamelCase_							)					,     [5, 6, 2, 5, 7, 8]							)
							self.assertListEqual(tokenizer.convert_tokens_to_shape_ids(UpperCamelCase_							)					,     [5, 6, 2, 5, 7, 8]							)
							self.assertListEqual(tokenizer.convert_tokens_to_pronunciation_ids(UpperCamelCase_							)					,     [5, 6, 2, 5, 7, 8]							)




		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							UpperCamelCase__     :int							      =				RoCBertBasicTokenizer()

							self.assertListEqual(tokenizer.tokenize('''ah\u535A\u63A8zz'''							)					,     ['''ah''', '''\u535A''', '''\u63A8''', '''zz''']							)




		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							UpperCamelCase__     :Optional[Any]							      =				RoCBertBasicTokenizer(do_lower_case=UpperCamelCase_							)

							self.assertListEqual(
							    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''							)					,     ['''hello''', '''!''', '''how''', '''are''', '''you''', '''?''']							)
							self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''							)					,     ['''hello''']							)




		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							UpperCamelCase__     :List[Any]							      =				RoCBertBasicTokenizer(do_lower_case=UpperCamelCase_					,     strip_accents=UpperCamelCase_							)

							self.assertListEqual(
							    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''							)					,     ['''hällo''', '''!''', '''how''', '''are''', '''you''', '''?''']							)
							self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''							)					,     ['''h\u00E9llo''']							)




		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							UpperCamelCase__     :str							      =				RoCBertBasicTokenizer(do_lower_case=UpperCamelCase_					,     strip_accents=UpperCamelCase_							)

							self.assertListEqual(
							    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''							)					,     ['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']							)
							self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''							)					,     ['''hello''']							)




		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							UpperCamelCase__     :Any							      =				RoCBertBasicTokenizer(do_lower_case=UpperCamelCase_							)

							self.assertListEqual(
							    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''							)					,     ['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']							)
							self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''							)					,     ['''hello''']							)




		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							UpperCamelCase__     :List[str]							      =				RoCBertBasicTokenizer(do_lower_case=UpperCamelCase_							)

							self.assertListEqual(
							    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''							)					,     ['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']							)




		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							UpperCamelCase__     :Union[str, Any]							      =				RoCBertBasicTokenizer(do_lower_case=UpperCamelCase_					,     strip_accents=UpperCamelCase_							)

							self.assertListEqual(
							    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''							)					,     ['''HäLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']							)




		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							UpperCamelCase__     :Dict							      =				RoCBertBasicTokenizer(do_lower_case=UpperCamelCase_					,     strip_accents=UpperCamelCase_							)

							self.assertListEqual(
							    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''							)					,     ['''HaLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']							)




		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							UpperCamelCase__     :str							      =				RoCBertBasicTokenizer(do_lower_case=UpperCamelCase_					,     never_split=['''[UNK]''']							)

							self.assertListEqual(
							    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU? [UNK]'''							)					,     ['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''', '''[UNK]''']							)




		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							UpperCamelCase__     :List[str]							      =				['''[UNK]''', '''[CLS]''', '''[SEP]''', '''want''', '''##want''', '''##ed''', '''wa''', '''un''', '''runn''', '''##ing''']

							UpperCamelCase__     :List[str]							      =				{}
							for i, token in enumerate(UpperCamelCase_							):
												UpperCamelCase__     :Tuple							      =				i
							UpperCamelCase__     :Any							      =				RoCBertWordpieceTokenizer(vocab=UpperCamelCase_					,     unk_token='''[UNK]'''							)

							self.assertListEqual(tokenizer.tokenize(''''''							)					,     []							)

							self.assertListEqual(tokenizer.tokenize('''unwanted running'''							)					,     ['''un''', '''##want''', '''##ed''', '''runn''', '''##ing''']							)

							self.assertListEqual(tokenizer.tokenize('''unwantedX running'''							)					,     ['''[UNK]''', '''runn''', '''##ing''']							)




		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							self.assertTrue(_is_whitespace(''' '''							)							)
							self.assertTrue(_is_whitespace('''\t'''							)							)
							self.assertTrue(_is_whitespace('''\r'''							)							)
							self.assertTrue(_is_whitespace('''\n'''							)							)
							self.assertTrue(_is_whitespace('''\u00A0'''							)							)

							self.assertFalse(_is_whitespace('''A'''							)							)
							self.assertFalse(_is_whitespace('''-'''							)							)




		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							self.assertTrue(_is_control('''\u0005'''							)							)

							self.assertFalse(_is_control('''A'''							)							)
							self.assertFalse(_is_control(''' '''							)							)
							self.assertFalse(_is_control('''\t'''							)							)
							self.assertFalse(_is_control('''\r'''							)							)




		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							self.assertTrue(_is_punctuation('''-'''							)							)
							self.assertTrue(_is_punctuation('''$'''							)							)
							self.assertTrue(_is_punctuation('''`'''							)							)
							self.assertTrue(_is_punctuation('''.'''							)							)

							self.assertFalse(_is_punctuation('''A'''							)							)
							self.assertFalse(_is_punctuation(''' '''							)							)




		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							UpperCamelCase__     :Union[str, Any]							      =				self.get_tokenizer()

							# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
							self.assertListEqual([tokenizer.tokenize(UpperCamelCase_							) for t in ['''Test''', '''\xad''', '''test''']]					,     [['''[UNK]'''], [], ['''[UNK]''']]							)

							if self.test_rust_tokenizer:
												UpperCamelCase__     :List[str]							      =				self.get_rust_tokenizer()
												self.assertListEqual(
												    [rust_tokenizer.tokenize(UpperCamelCase_							) for t in ['''Test''', '''\xad''', '''test''']]					,     [['''[UNK]'''], [], ['''[UNK]''']]							)




		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
												with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''							):
																	UpperCamelCase__     :Optional[Any]							      =				self.rust_tokenizer_class.from_pretrained(UpperCamelCase_					,     **UpperCamelCase_							)

																	UpperCamelCase__     :Optional[int]							      =				F'''A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'''
																	UpperCamelCase__     :List[str]							      =				tokenizer_r.encode_plus(
																	    UpperCamelCase_					,     return_attention_mask=UpperCamelCase_					,     return_token_type_ids=UpperCamelCase_					,     return_offsets_mapping=UpperCamelCase_					,     add_special_tokens=UpperCamelCase_					,     )

																	UpperCamelCase__     :int							      =				tokenizer_r.do_lower_case if hasattr(UpperCamelCase_					,     '''do_lower_case'''							) else False
																	UpperCamelCase__     :str							      =				(
																	    [
																	        ((0, 0), tokenizer_r.cls_token),
																	        ((0, 1), '''A'''),
																	        ((1, 2), ''','''),
																	        ((3, 5), '''na'''),
																	        ((5, 6), '''##ï'''),
																	        ((6, 8), '''##ve'''),
																	        ((9, 15), tokenizer_r.mask_token),
																	        ((16, 21), '''Allen'''),
																	        ((21, 23), '''##NL'''),
																	        ((23, 24), '''##P'''),
																	        ((25, 33), '''sentence'''),
																	        ((33, 34), '''.'''),
																	        ((0, 0), tokenizer_r.sep_token),
																	    ]
																	    if not do_lower_case
																	    else [
																	        ((0, 0), tokenizer_r.cls_token),
																	        ((0, 1), '''a'''),
																	        ((1, 2), ''','''),
																	        ((3, 8), '''naive'''),
																	        ((9, 15), tokenizer_r.mask_token),
																	        ((16, 21), '''allen'''),
																	        ((21, 23), '''##nl'''),
																	        ((23, 24), '''##p'''),
																	        ((25, 33), '''sentence'''),
																	        ((33, 34), '''.'''),
																	        ((0, 0), tokenizer_r.sep_token),
																	    ]
																	)

																	self.assertEqual(
																	    [e[1] for e in expected_results]					,     tokenizer_r.convert_ids_to_tokens(tokens['''input_ids''']							)							)
																	self.assertEqual([e[0] for e in expected_results]					,     tokens['''offset_mapping''']							)




		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							UpperCamelCase__     :Optional[Any]							      =				['''的''', '''人''', '''有''']
							UpperCamelCase__     :Tuple							      =				''''''.join(UpperCamelCase_							)
							for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
												with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''							):
																	UpperCamelCase__     :Union[str, Any]							      =				True
																	UpperCamelCase__     :str							      =				self.tokenizer_class.from_pretrained(UpperCamelCase_					,     **UpperCamelCase_							)
																	UpperCamelCase__     :Any							      =				self.rust_tokenizer_class.from_pretrained(UpperCamelCase_					,     **UpperCamelCase_							)

																	UpperCamelCase__     :Union[str, Any]							      =				tokenizer_p.encode(UpperCamelCase_					,     add_special_tokens=UpperCamelCase_							)
																	UpperCamelCase__     :Optional[Any]							      =				tokenizer_r.encode(UpperCamelCase_					,     add_special_tokens=UpperCamelCase_							)

																	UpperCamelCase__     :Any							      =				tokenizer_r.convert_ids_to_tokens(UpperCamelCase_							)
																	UpperCamelCase__     :str							      =				tokenizer_p.convert_ids_to_tokens(UpperCamelCase_							)

																	# it is expected that each Chinese character is not preceded by "##"
																	self.assertListEqual(UpperCamelCase_					,     UpperCamelCase_							)
																	self.assertListEqual(UpperCamelCase_					,     UpperCamelCase_							)

																	UpperCamelCase__     :Optional[int]							      =				False
																	UpperCamelCase__     :Union[str, Any]							      =				self.rust_tokenizer_class.from_pretrained(UpperCamelCase_					,     **UpperCamelCase_							)
																	UpperCamelCase__     :Dict							      =				self.tokenizer_class.from_pretrained(UpperCamelCase_					,     **UpperCamelCase_							)

																	UpperCamelCase__     :Optional[int]							      =				tokenizer_r.encode(UpperCamelCase_					,     add_special_tokens=UpperCamelCase_							)
																	UpperCamelCase__     :List[Any]							      =				tokenizer_p.encode(UpperCamelCase_					,     add_special_tokens=UpperCamelCase_							)

																	UpperCamelCase__     :Any							      =				tokenizer_r.convert_ids_to_tokens(UpperCamelCase_							)
																	UpperCamelCase__     :int							      =				tokenizer_p.convert_ids_to_tokens(UpperCamelCase_							)

																	# it is expected that only the first Chinese character is not preceded by "##".
																	UpperCamelCase__     :Optional[Any]							      =				[
																	    F'''##{token}''' if idx != 0 else token for idx, token in enumerate(UpperCamelCase_							)
																	]
																	self.assertListEqual(UpperCamelCase_					,     UpperCamelCase_							)
																	self.assertListEqual(UpperCamelCase_					,     UpperCamelCase_							)




		@slow
		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							UpperCamelCase__     :int							      =				self.tokenizer_class(self.vocab_file					,     self.word_shape_file					,     self.word_pronunciation_file							)

							UpperCamelCase__     :Union[str, Any]							      =				tokenizer.encode('''你好'''					,     add_special_tokens=UpperCamelCase_							)
							UpperCamelCase__     :Dict							      =				tokenizer.encode('''你是谁'''					,     add_special_tokens=UpperCamelCase_							)

							UpperCamelCase__     :Union[str, Any]							      =				tokenizer.build_inputs_with_special_tokens(UpperCamelCase_							)
							UpperCamelCase__     :Optional[int]							      =				tokenizer.build_inputs_with_special_tokens(UpperCamelCase_					,     UpperCamelCase_							)

							assert encoded_sentence == [1] + text + [2]
							assert encoded_pair == [1] + text + [2] + text_a + [2]




		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							UpperCamelCase__     :int							      =				self.get_tokenizers(do_lower_case=UpperCamelCase_							)
							for tokenizer in tokenizers:
												with self.subTest(F'''{tokenizer.__class__.__name__}'''							):
																	UpperCamelCase__     :Optional[int]							      =				'''你好，你是谁'''
																	UpperCamelCase__     :Optional[int]							      =				tokenizer.tokenize(UpperCamelCase_							)
																	UpperCamelCase__     :Union[str, Any]							      =				tokenizer.convert_tokens_to_ids(UpperCamelCase_							)
																	UpperCamelCase__     :Any							      =				tokenizer.convert_tokens_to_shape_ids(UpperCamelCase_							)
																	UpperCamelCase__     :int							      =				tokenizer.convert_tokens_to_pronunciation_ids(UpperCamelCase_							)
																	UpperCamelCase__     :Optional[int]							      =				tokenizer.prepare_for_model(
																	    UpperCamelCase_					,     UpperCamelCase_					,     UpperCamelCase_					,     add_special_tokens=UpperCamelCase_							)

																	UpperCamelCase__     :Optional[int]							      =				tokenizer.encode_plus(UpperCamelCase_					,     add_special_tokens=UpperCamelCase_							)

																	self.assertEqual(UpperCamelCase_					,     UpperCamelCase_							)