






"""simple docstring"""






import json
import os
import unittest

from transformers.models.roc_bert.tokenization_roc_bert import (
    VOCAB_FILES_NAMES,
    RoCBertBasicTokenizer,
    RoCBertTokenizer,
    RoCBertWordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english





@require_tokenizers
class       snake_case_       (    _lowerCamelCase    ,							unittest.TestCase       ):



	"""simple docstring"""

	SCREAMING_SNAKE_CASE_:							List[Any]   			=					RoCBertTokenizer
	SCREAMING_SNAKE_CASE_:							Optional[Any]   			=					None
	SCREAMING_SNAKE_CASE_:							Dict   			=					False
	SCREAMING_SNAKE_CASE_:							List[str]   			=					True
	SCREAMING_SNAKE_CASE_:							str   			=					filter_non_english


	def       _UpperCAmelCase						(      self						):


					"""simple docstring"""

					super().setUp()

					A__							 =       ['[UNK]', '[CLS]', '[SEP]', '[PAD]', '[MASK]', '你', '好', '是', '谁', 'a', 'b', 'c', 'd']
					A__							 =       {}
					A__							 =       {}
					for i, value in enumerate(__a						):
									A__							 =       i
									A__							 =       i
					A__							 =       os.path.join(self.tmpdirname  ,					VOCAB_FILES_NAMES['vocab_file']						)
					A__							 =       os.path.join(self.tmpdirname  ,					VOCAB_FILES_NAMES['word_shape_file']						)
					A__							 =       os.path.join(self.tmpdirname  ,					VOCAB_FILES_NAMES['word_pronunciation_file']						)
					with open(self.vocab_file  ,					'w'  ,					encoding='utf-8'						) as vocab_writer:
									vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]						)						)
					with open(self.word_shape_file  ,					'w'  ,					encoding='utf-8'						) as word_shape_writer:
									json.dump(__a  ,					__a  ,					ensure_ascii=__a						)
					with open(self.word_pronunciation_file  ,					'w'  ,					encoding='utf-8'						) as word_pronunciation_writer:
									json.dump(__a  ,					__a  ,					ensure_ascii=__a						)


	def       _UpperCAmelCase						(      self						):


					"""simple docstring"""

					A__							 =       self.tokenizer_class(self.vocab_file  ,					self.word_shape_file  ,					self.word_pronunciation_file						)

					A__							 =       tokenizer.tokenize('你好[SEP]你是谁'						)
					self.assertListEqual(__a  ,					['你', '好', '[SEP]', '你', '是', '谁']						)
					self.assertListEqual(tokenizer.convert_tokens_to_ids(__a						)  ,					[5, 6, 2, 5, 7, 8]						)
					self.assertListEqual(tokenizer.convert_tokens_to_shape_ids(__a						)  ,					[5, 6, 2, 5, 7, 8]						)
					self.assertListEqual(tokenizer.convert_tokens_to_pronunciation_ids(__a						)  ,					[5, 6, 2, 5, 7, 8]						)


	def       _UpperCAmelCase						(      self						):


					"""simple docstring"""

					A__							 =       RoCBertBasicTokenizer()

					self.assertListEqual(tokenizer.tokenize('ah\u535A\u63A8zz'						)  ,					['ah', '\u535A', '\u63A8', 'zz']						)


	def       _UpperCAmelCase						(      self						):


					"""simple docstring"""

					A__							 =       RoCBertBasicTokenizer(do_lower_case=__a						)

					self.assertListEqual(
					    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '						)  ,					['hello', '!', 'how', 'are', 'you', '?']						)
					self.assertListEqual(tokenizer.tokenize('H\u00E9llo'						)  ,					['hello']						)


	def       _UpperCAmelCase						(      self						):


					"""simple docstring"""

					A__							 =       RoCBertBasicTokenizer(do_lower_case=__a  ,					strip_accents=__a						)

					self.assertListEqual(
					    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '						)  ,					['hällo', '!', 'how', 'are', 'you', '?']						)
					self.assertListEqual(tokenizer.tokenize('H\u00E9llo'						)  ,					['h\u00E9llo']						)


	def       _UpperCAmelCase						(      self						):


					"""simple docstring"""

					A__							 =       RoCBertBasicTokenizer(do_lower_case=__a  ,					strip_accents=__a						)

					self.assertListEqual(
					    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '						)  ,					['hallo', '!', 'how', 'are', 'you', '?']						)
					self.assertListEqual(tokenizer.tokenize('H\u00E9llo'						)  ,					['hello']						)


	def       _UpperCAmelCase						(      self						):


					"""simple docstring"""

					A__							 =       RoCBertBasicTokenizer(do_lower_case=__a						)

					self.assertListEqual(
					    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '						)  ,					['hallo', '!', 'how', 'are', 'you', '?']						)
					self.assertListEqual(tokenizer.tokenize('H\u00E9llo'						)  ,					['hello']						)


	def       _UpperCAmelCase						(      self						):


					"""simple docstring"""

					A__							 =       RoCBertBasicTokenizer(do_lower_case=__a						)

					self.assertListEqual(
					    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '						)  ,					['HeLLo', '!', 'how', 'Are', 'yoU', '?']						)


	def       _UpperCAmelCase						(      self						):


					"""simple docstring"""

					A__							 =       RoCBertBasicTokenizer(do_lower_case=__a  ,					strip_accents=__a						)

					self.assertListEqual(
					    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '						)  ,					['HäLLo', '!', 'how', 'Are', 'yoU', '?']						)


	def       _UpperCAmelCase						(      self						):


					"""simple docstring"""

					A__							 =       RoCBertBasicTokenizer(do_lower_case=__a  ,					strip_accents=__a						)

					self.assertListEqual(
					    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '						)  ,					['HaLLo', '!', 'how', 'Are', 'yoU', '?']						)


	def       _UpperCAmelCase						(      self						):


					"""simple docstring"""

					A__							 =       RoCBertBasicTokenizer(do_lower_case=__a  ,					never_split=['[UNK]']						)

					self.assertListEqual(
					    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU? [UNK]'						)  ,					['HeLLo', '!', 'how', 'Are', 'yoU', '?', '[UNK]']						)


	def       _UpperCAmelCase						(      self						):


					"""simple docstring"""

					A__							 =       ['[UNK]', '[CLS]', '[SEP]', 'want', '##want', '##ed', 'wa', 'un', 'runn', '##ing']

					A__							 =       {}
					for i, token in enumerate(__a						):
									A__							 =       i
					A__							 =       RoCBertWordpieceTokenizer(vocab=__a  ,					unk_token='[UNK]'						)

					self.assertListEqual(tokenizer.tokenize(''						)  ,					[]						)

					self.assertListEqual(tokenizer.tokenize('unwanted running'						)  ,					['un', '##want', '##ed', 'runn', '##ing']						)

					self.assertListEqual(tokenizer.tokenize('unwantedX running'						)  ,					['[UNK]', 'runn', '##ing']						)


	def       _UpperCAmelCase						(      self						):


					"""simple docstring"""

					self.assertTrue(_is_whitespace(' '						)						)
					self.assertTrue(_is_whitespace('\t'						)						)
					self.assertTrue(_is_whitespace('\r'						)						)
					self.assertTrue(_is_whitespace('\n'						)						)
					self.assertTrue(_is_whitespace('\u00A0'						)						)

					self.assertFalse(_is_whitespace('A'						)						)
					self.assertFalse(_is_whitespace('-'						)						)


	def       _UpperCAmelCase						(      self						):


					"""simple docstring"""

					self.assertTrue(_is_control('\u0005'						)						)

					self.assertFalse(_is_control('A'						)						)
					self.assertFalse(_is_control(' '						)						)
					self.assertFalse(_is_control('\t'						)						)
					self.assertFalse(_is_control('\r'						)						)


	def       _UpperCAmelCase						(      self						):


					"""simple docstring"""

					self.assertTrue(_is_punctuation('-'						)						)
					self.assertTrue(_is_punctuation('$'						)						)
					self.assertTrue(_is_punctuation('`'						)						)
					self.assertTrue(_is_punctuation('.'						)						)

					self.assertFalse(_is_punctuation('A'						)						)
					self.assertFalse(_is_punctuation(' '						)						)


	def       _UpperCAmelCase						(      self						):


					"""simple docstring"""

					A__							 =       self.get_tokenizer()

					# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
					self.assertListEqual([tokenizer.tokenize(__a						) for t in ['Test', '\xad', 'test']]  ,					[['[UNK]'], [], ['[UNK]']]						)

					if self.test_rust_tokenizer:
									A__							 =       self.get_rust_tokenizer()
									self.assertListEqual(
									    [rust_tokenizer.tokenize(__a						) for t in ['Test', '\xad', 'test']]  ,					[['[UNK]'], [], ['[UNK]']]						)


	def       _UpperCAmelCase						(      self						):


					"""simple docstring"""

					for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
									with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''						):
													A__							 =       self.rust_tokenizer_class.from_pretrained(__a  ,					**__a						)

													A__							 =       f'''A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'''
													A__							 =       tokenizer_r.encode_plus(
													    __a  ,					return_attention_mask=__a  ,					return_token_type_ids=__a  ,					return_offsets_mapping=__a  ,					add_special_tokens=__a  ,					)

													A__							 =       tokenizer_r.do_lower_case if hasattr(__a  ,					'do_lower_case'						) else False
													A__							 =       (
													    [
													        ((0, 0), tokenizer_r.cls_token),
													        ((0, 1), 'A'),
													        ((1, 2), ','),
													        ((3, 5), 'na'),
													        ((5, 6), '##ï'),
													        ((6, 8), '##ve'),
													        ((9, 15), tokenizer_r.mask_token),
													        ((16, 21), 'Allen'),
													        ((21, 23), '##NL'),
													        ((23, 24), '##P'),
													        ((25, 33), 'sentence'),
													        ((33, 34), '.'),
													        ((0, 0), tokenizer_r.sep_token),
													    ]
													    if not do_lower_case
													    else [
													        ((0, 0), tokenizer_r.cls_token),
													        ((0, 1), 'a'),
													        ((1, 2), ','),
													        ((3, 8), 'naive'),
													        ((9, 15), tokenizer_r.mask_token),
													        ((16, 21), 'allen'),
													        ((21, 23), '##nl'),
													        ((23, 24), '##p'),
													        ((25, 33), 'sentence'),
													        ((33, 34), '.'),
													        ((0, 0), tokenizer_r.sep_token),
													    ]
													)

													self.assertEqual(
													    [e[1] for e in expected_results]  ,					tokenizer_r.convert_ids_to_tokens(tokens['input_ids']						)						)
													self.assertEqual([e[0] for e in expected_results]  ,					tokens['offset_mapping']						)


	def       _UpperCAmelCase						(      self						):


					"""simple docstring"""

					A__							 =       ['的', '人', '有']
					A__							 =       ''.join(__a						)
					for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
									with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''						):
													A__							 =       True
													A__							 =       self.tokenizer_class.from_pretrained(__a  ,					**__a						)
													A__							 =       self.rust_tokenizer_class.from_pretrained(__a  ,					**__a						)

													A__							 =       tokenizer_p.encode(__a  ,					add_special_tokens=__a						)
													A__							 =       tokenizer_r.encode(__a  ,					add_special_tokens=__a						)

													A__							 =       tokenizer_r.convert_ids_to_tokens(__a						)
													A__							 =       tokenizer_p.convert_ids_to_tokens(__a						)

													# it is expected that each Chinese character is not preceded by "##"
													self.assertListEqual(__a  ,					__a						)
													self.assertListEqual(__a  ,					__a						)

													A__							 =       False
													A__							 =       self.rust_tokenizer_class.from_pretrained(__a  ,					**__a						)
													A__							 =       self.tokenizer_class.from_pretrained(__a  ,					**__a						)

													A__							 =       tokenizer_r.encode(__a  ,					add_special_tokens=__a						)
													A__							 =       tokenizer_p.encode(__a  ,					add_special_tokens=__a						)

													A__							 =       tokenizer_r.convert_ids_to_tokens(__a						)
													A__							 =       tokenizer_p.convert_ids_to_tokens(__a						)

													# it is expected that only the first Chinese character is not preceded by "##".
													A__							 =       [
													    f'''##{token}''' if idx != 0 else token for idx, token in enumerate(__a						)
													]
													self.assertListEqual(__a  ,					__a						)
													self.assertListEqual(__a  ,					__a						)


	@slow
	def       _UpperCAmelCase						(      self						):


					"""simple docstring"""

					A__							 =       self.tokenizer_class(self.vocab_file  ,					self.word_shape_file  ,					self.word_pronunciation_file						)

					A__							 =       tokenizer.encode('你好'  ,					add_special_tokens=__a						)
					A__							 =       tokenizer.encode('你是谁'  ,					add_special_tokens=__a						)

					A__							 =       tokenizer.build_inputs_with_special_tokens(__a						)
					A__							 =       tokenizer.build_inputs_with_special_tokens(__a  ,					__a						)

					assert encoded_sentence == [1] + text + [2]
					assert encoded_pair == [1] + text + [2] + text_a + [2]






	def       _UpperCAmelCase						(      self						):


					"""simple docstring"""

					A__							 =       self.get_tokenizers(do_lower_case=__a						)
					for tokenizer in tokenizers:
									with self.subTest(f'''{tokenizer.__class__.__name__}'''						):
													A__							 =       '你好，你是谁'
													A__							 =       tokenizer.tokenize(__a						)
													A__							 =       tokenizer.convert_tokens_to_ids(__a						)
													A__							 =       tokenizer.convert_tokens_to_shape_ids(__a						)
													A__							 =       tokenizer.convert_tokens_to_pronunciation_ids(__a						)
													A__							 =       tokenizer.prepare_for_model(
													    __a  ,					__a  ,					__a  ,					add_special_tokens=__a						)

													A__							 =       tokenizer.encode_plus(__a  ,					add_special_tokens=__a						)

													self.assertEqual(__a  ,					__a						)










"""simple docstring"""






from ...processing_utils import ProcessorMixin





class       snake_case_       (    _lowerCamelCase       ):



	"""simple docstring"""

	SCREAMING_SNAKE_CASE_:							Union[str, Any]   			=					["""image_processor""", """feature_extractor"""]
	SCREAMING_SNAKE_CASE_:							Optional[int]   			=					"""TvltImageProcessor"""
	SCREAMING_SNAKE_CASE_:							Optional[int]   			=					"""TvltFeatureExtractor"""


	def __init__(      self  ,					__a  ,					__a						):


					"""simple docstring"""

					super().__init__(image_processor=__a  ,					feature_extractor=__a						)

					A__							 =       image_processor
					A__							 =       feature_extractor


	def __call__(      self  ,					__a=None  ,					__a=None  ,					__a=None  ,					__a=None  ,					__a=False  ,					__a=False  ,					*__a  ,					**__a  ,					):


					"""simple docstring"""

					if images is None and audio is None:
									raise ValueError('You need to specify either an `images` or `audio` input to process.'						)

					A__							 =       None
					if images is not None:
									A__							 =       self.image_processor(__a  ,					mask_pixel=__a  ,					*__a  ,					**__a						)
					if images_mixed is not None:
									A__							 =       self.image_processor(__a  ,					is_mixed=__a  ,					*__a  ,					**__a						)
					if audio is not None:
									A__							 =       self.feature_extractor(
									    __a  ,					*__a  ,					sampling_rate=__a  ,					mask_audio=__a  ,					**__a						)

					A__							 =       {}
					if audio is not None:
									output_dict.update(__a						)
					if images is not None:
									output_dict.update(__a						)
					if images_mixed_dict is not None:
									output_dict.update(__a						)
					return output_dict






	@property
	def       _UpperCAmelCase						(      self						):


					"""simple docstring"""

					A__							 =       self.image_processor.model_input_names
					A__							 =       self.feature_extractor.model_input_names
					return list(dict.fromkeys(image_processor_input_names + feature_extractor_input_names						)						)


