



"""simple docstring"""







import json
import os
import unittest

from transformers.models.roc_bert.tokenization_roc_bert import (
    VOCAB_FILES_NAMES,
    RoCBertBasicTokenizer,
    RoCBertTokenizer,
    RoCBertWordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english






@require_tokenizers
class      UpperCAmelCase__		(      UpperCamelCase   ,unittest.TestCase     ):
		lowerCAmelCase_       :	str 							=     RoCBertTokenizer
		lowerCAmelCase_       :	Any 							=     None
		lowerCAmelCase_       :	Optional[int] 							=     False
		lowerCAmelCase_       :	Optional[Any] 							=     True
		lowerCAmelCase_       :	Tuple 							=     filter_non_english



		def 					A_		( self  :    Tuple				)			->						Tuple:






									'''simple docstring'''

									super().setUp()

									A         =					['[UNK]', '[CLS]', '[SEP]', '[PAD]', '[MASK]', '你', '好', '是', '谁', 'a', 'b', 'c', 'd']
									A         =					{}
									A         =					{}
									for i, value in enumerate(snake_case				):
																A         =					i
																A         =					i
									A         =					os.path.join(self.tmpdirname      ,							VOCAB_FILES_NAMES['vocab_file']				)
									A         =					os.path.join(self.tmpdirname      ,							VOCAB_FILES_NAMES['word_shape_file']				)
									A         =					os.path.join(self.tmpdirname      ,							VOCAB_FILES_NAMES['word_pronunciation_file']				)
									with open(self.vocab_file      ,							'w'      ,							encoding='utf-8'				) as vocab_writer:
																vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]				)				)
									with open(self.word_shape_file      ,							'w'      ,							encoding='utf-8'				) as word_shape_writer:
																json.dump(snake_case      ,							snake_case      ,							ensure_ascii=snake_case				)
									with open(self.word_pronunciation_file      ,							'w'      ,							encoding='utf-8'				) as word_pronunciation_writer:
																json.dump(snake_case      ,							snake_case      ,							ensure_ascii=snake_case				)



		def 					A_		( self  :    List[str]				)			->						str:






									'''simple docstring'''

									A         =					self.tokenizer_class(self.vocab_file      ,							self.word_shape_file      ,							self.word_pronunciation_file				)

									A         =					tokenizer.tokenize('你好[SEP]你是谁'				)
									self.assertListEqual(snake_case      ,							['你', '好', '[SEP]', '你', '是', '谁']				)
									self.assertListEqual(tokenizer.convert_tokens_to_ids(snake_case				)      ,							[5, 6, 2, 5, 7, 8]				)
									self.assertListEqual(tokenizer.convert_tokens_to_shape_ids(snake_case				)      ,							[5, 6, 2, 5, 7, 8]				)
									self.assertListEqual(tokenizer.convert_tokens_to_pronunciation_ids(snake_case				)      ,							[5, 6, 2, 5, 7, 8]				)



		def 					A_		( self  :    str				)			->						Tuple:






									'''simple docstring'''

									A         =					RoCBertBasicTokenizer()

									self.assertListEqual(tokenizer.tokenize('ah\u535A\u63A8zz'				)      ,							['ah', '\u535A', '\u63A8', 'zz']				)



		def 					A_		( self  :    int				)			->						Any:






									'''simple docstring'''

									A         =					RoCBertBasicTokenizer(do_lower_case=snake_case				)

									self.assertListEqual(
									    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '				)      ,							['hello', '!', 'how', 'are', 'you', '?']				)
									self.assertListEqual(tokenizer.tokenize('H\u00E9llo'				)      ,							['hello']				)



		def 					A_		( self  :    str				)			->						str:






									'''simple docstring'''

									A         =					RoCBertBasicTokenizer(do_lower_case=snake_case      ,							strip_accents=snake_case				)

									self.assertListEqual(
									    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '				)      ,							['hällo', '!', 'how', 'are', 'you', '?']				)
									self.assertListEqual(tokenizer.tokenize('H\u00E9llo'				)      ,							['h\u00E9llo']				)



		def 					A_		( self  :    List[Any]				)			->						Tuple:






									'''simple docstring'''

									A         =					RoCBertBasicTokenizer(do_lower_case=snake_case      ,							strip_accents=snake_case				)

									self.assertListEqual(
									    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '				)      ,							['hallo', '!', 'how', 'are', 'you', '?']				)
									self.assertListEqual(tokenizer.tokenize('H\u00E9llo'				)      ,							['hello']				)



		def 					A_		( self  :    List[str]				)			->						Optional[int]:






									'''simple docstring'''

									A         =					RoCBertBasicTokenizer(do_lower_case=snake_case				)

									self.assertListEqual(
									    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '				)      ,							['hallo', '!', 'how', 'are', 'you', '?']				)
									self.assertListEqual(tokenizer.tokenize('H\u00E9llo'				)      ,							['hello']				)



		def 					A_		( self  :    Tuple				)			->						Any:






									'''simple docstring'''

									A         =					RoCBertBasicTokenizer(do_lower_case=snake_case				)

									self.assertListEqual(
									    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '				)      ,							['HeLLo', '!', 'how', 'Are', 'yoU', '?']				)



		def 					A_		( self  :    List[Any]				)			->						Any:






									'''simple docstring'''

									A         =					RoCBertBasicTokenizer(do_lower_case=snake_case      ,							strip_accents=snake_case				)

									self.assertListEqual(
									    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '				)      ,							['HäLLo', '!', 'how', 'Are', 'yoU', '?']				)



		def 					A_		( self  :    List[Any]				)			->						int:






									'''simple docstring'''

									A         =					RoCBertBasicTokenizer(do_lower_case=snake_case      ,							strip_accents=snake_case				)

									self.assertListEqual(
									    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '				)      ,							['HaLLo', '!', 'how', 'Are', 'yoU', '?']				)



		def 					A_		( self  :    str				)			->						Dict:






									'''simple docstring'''

									A         =					RoCBertBasicTokenizer(do_lower_case=snake_case      ,							never_split=['[UNK]']				)

									self.assertListEqual(
									    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU? [UNK]'				)      ,							['HeLLo', '!', 'how', 'Are', 'yoU', '?', '[UNK]']				)



		def 					A_		( self  :    Optional[Any]				)			->						List[str]:






									'''simple docstring'''

									A         =					['[UNK]', '[CLS]', '[SEP]', 'want', '##want', '##ed', 'wa', 'un', 'runn', '##ing']

									A         =					{}
									for i, token in enumerate(snake_case				):
																A         =					i
									A         =					RoCBertWordpieceTokenizer(vocab=snake_case      ,							unk_token='[UNK]'				)

									self.assertListEqual(tokenizer.tokenize(''				)      ,							[]				)

									self.assertListEqual(tokenizer.tokenize('unwanted running'				)      ,							['un', '##want', '##ed', 'runn', '##ing']				)

									self.assertListEqual(tokenizer.tokenize('unwantedX running'				)      ,							['[UNK]', 'runn', '##ing']				)



		def 					A_		( self  :    Optional[Any]				)			->						Optional[Any]:






									'''simple docstring'''

									self.assertTrue(_is_whitespace(' '				)				)
									self.assertTrue(_is_whitespace('\t'				)				)
									self.assertTrue(_is_whitespace('\r'				)				)
									self.assertTrue(_is_whitespace('\n'				)				)
									self.assertTrue(_is_whitespace('\u00A0'				)				)

									self.assertFalse(_is_whitespace('A'				)				)
									self.assertFalse(_is_whitespace('-'				)				)



		def 					A_		( self  :    List[str]				)			->						Dict:






									'''simple docstring'''

									self.assertTrue(_is_control('\u0005'				)				)

									self.assertFalse(_is_control('A'				)				)
									self.assertFalse(_is_control(' '				)				)
									self.assertFalse(_is_control('\t'				)				)
									self.assertFalse(_is_control('\r'				)				)



		def 					A_		( self  :    int				)			->						List[Any]:






									'''simple docstring'''

									self.assertTrue(_is_punctuation('-'				)				)
									self.assertTrue(_is_punctuation('$'				)				)
									self.assertTrue(_is_punctuation('`'				)				)
									self.assertTrue(_is_punctuation('.'				)				)

									self.assertFalse(_is_punctuation('A'				)				)
									self.assertFalse(_is_punctuation(' '				)				)



		def 					A_		( self  :    Tuple				)			->						Tuple:






									'''simple docstring'''

									A         =					self.get_tokenizer()

									# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
									self.assertListEqual([tokenizer.tokenize(snake_case				) for t in ['Test', '\xad', 'test']]      ,							[['[UNK]'], [], ['[UNK]']]				)

									if self.test_rust_tokenizer:
																A         =					self.get_rust_tokenizer()
																self.assertListEqual(
																    [rust_tokenizer.tokenize(snake_case				) for t in ['Test', '\xad', 'test']]      ,							[['[UNK]'], [], ['[UNK]']]				)



		def 					A_		( self  :    Optional[int]				)			->						Optional[Any]:






									'''simple docstring'''

									for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""				):
																							A         =					self.rust_tokenizer_class.from_pretrained(snake_case      ,							**snake_case				)

																							A         =					f"""A, naïve {tokenizer_r.mask_token} AllenNLP sentence."""
																							A         =					tokenizer_r.encode_plus(
																							    snake_case      ,							return_attention_mask=snake_case      ,							return_token_type_ids=snake_case      ,							return_offsets_mapping=snake_case      ,							add_special_tokens=snake_case      ,							)

																							A         =					tokenizer_r.do_lower_case if hasattr(snake_case      ,							'do_lower_case'				) else False
																							A         =					(
																							    [
																							        ((0, 0), tokenizer_r.cls_token),
																							        ((0, 1), 'A'),
																							        ((1, 2), ','),
																							        ((3, 5), 'na'),
																							        ((5, 6), '##ï'),
																							        ((6, 8), '##ve'),
																							        ((9, 15), tokenizer_r.mask_token),
																							        ((16, 21), 'Allen'),
																							        ((21, 23), '##NL'),
																							        ((23, 24), '##P'),
																							        ((25, 33), 'sentence'),
																							        ((33, 34), '.'),
																							        ((0, 0), tokenizer_r.sep_token),
																							    ]
																							    if not do_lower_case
																							    else [
																							        ((0, 0), tokenizer_r.cls_token),
																							        ((0, 1), 'a'),
																							        ((1, 2), ','),
																							        ((3, 8), 'naive'),
																							        ((9, 15), tokenizer_r.mask_token),
																							        ((16, 21), 'allen'),
																							        ((21, 23), '##nl'),
																							        ((23, 24), '##p'),
																							        ((25, 33), 'sentence'),
																							        ((33, 34), '.'),
																							        ((0, 0), tokenizer_r.sep_token),
																							    ]
																							)

																							self.assertEqual(
																							    [e[1] for e in expected_results]      ,							tokenizer_r.convert_ids_to_tokens(tokens['input_ids']				)				)
																							self.assertEqual([e[0] for e in expected_results]      ,							tokens['offset_mapping']				)



		def 					A_		( self  :    int				)			->						int:






									'''simple docstring'''

									A         =					['的', '人', '有']
									A         =					''.join(snake_case				)
									for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""				):
																							A         =					True
																							A         =					self.tokenizer_class.from_pretrained(snake_case      ,							**snake_case				)
																							A         =					self.rust_tokenizer_class.from_pretrained(snake_case      ,							**snake_case				)

																							A         =					tokenizer_p.encode(snake_case      ,							add_special_tokens=snake_case				)
																							A         =					tokenizer_r.encode(snake_case      ,							add_special_tokens=snake_case				)

																							A         =					tokenizer_r.convert_ids_to_tokens(snake_case				)
																							A         =					tokenizer_p.convert_ids_to_tokens(snake_case				)

																							# it is expected that each Chinese character is not preceded by "##"
																							self.assertListEqual(snake_case      ,							snake_case				)
																							self.assertListEqual(snake_case      ,							snake_case				)

																							A         =					False
																							A         =					self.rust_tokenizer_class.from_pretrained(snake_case      ,							**snake_case				)
																							A         =					self.tokenizer_class.from_pretrained(snake_case      ,							**snake_case				)

																							A         =					tokenizer_r.encode(snake_case      ,							add_special_tokens=snake_case				)
																							A         =					tokenizer_p.encode(snake_case      ,							add_special_tokens=snake_case				)

																							A         =					tokenizer_r.convert_ids_to_tokens(snake_case				)
																							A         =					tokenizer_p.convert_ids_to_tokens(snake_case				)

																							# it is expected that only the first Chinese character is not preceded by "##".
																							A         =					[
																							    f"""##{token}""" if idx != 0 else token for idx, token in enumerate(snake_case				)
																							]
																							self.assertListEqual(snake_case      ,							snake_case				)
																							self.assertListEqual(snake_case      ,							snake_case				)



		@slow
		def 					A_		( self  :    Tuple				)			->						Dict:






									'''simple docstring'''

									A         =					self.tokenizer_class(self.vocab_file      ,							self.word_shape_file      ,							self.word_pronunciation_file				)

									A         =					tokenizer.encode('你好'      ,							add_special_tokens=snake_case				)
									A         =					tokenizer.encode('你是谁'      ,							add_special_tokens=snake_case				)

									A         =					tokenizer.build_inputs_with_special_tokens(snake_case				)
									A         =					tokenizer.build_inputs_with_special_tokens(snake_case      ,							snake_case				)

									assert encoded_sentence == [1] + text + [2]
									assert encoded_pair == [1] + text + [2] + text_a + [2]



		def 					A_		( self  :    Optional[Any]				)			->						Any:






									'''simple docstring'''

									A         =					self.get_tokenizers(do_lower_case=snake_case				)
									for tokenizer in tokenizers:
																with self.subTest(f"""{tokenizer.__class__.__name__}"""				):
																							A         =					'你好，你是谁'
																							A         =					tokenizer.tokenize(snake_case				)
																							A         =					tokenizer.convert_tokens_to_ids(snake_case				)
																							A         =					tokenizer.convert_tokens_to_shape_ids(snake_case				)
																							A         =					tokenizer.convert_tokens_to_pronunciation_ids(snake_case				)
																							A         =					tokenizer.prepare_for_model(
																							    snake_case      ,							snake_case      ,							snake_case      ,							add_special_tokens=snake_case				)

																							A         =					tokenizer.encode_plus(snake_case      ,							add_special_tokens=snake_case				)

																							self.assertEqual(snake_case      ,							snake_case				)








"""simple docstring"""







from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_sentencepiece_available,
    is_tokenizers_available,
    is_torch_available,
)


A    =			{'configuration_plbart': ['PLBART_PRETRAINED_CONFIG_ARCHIVE_MAP', 'PLBartConfig']}

try:
							if not is_sentencepiece_available():
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							pass
else:
							A    =			['PLBartTokenizer']

try:
							if not is_torch_available():
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							pass
else:
							A    =			[
							    'PLBART_PRETRAINED_MODEL_ARCHIVE_LIST',
							    'PLBartForCausalLM',
							    'PLBartForConditionalGeneration',
							    'PLBartForSequenceClassification',
							    'PLBartModel',
							    'PLBartPreTrainedModel',
							]


if TYPE_CHECKING:
							from .configuration_plbart import PLBART_PRETRAINED_CONFIG_ARCHIVE_MAP, PLBartConfig

							try:
														if not is_sentencepiece_available():
																					raise OptionalDependencyNotAvailable()
							except OptionalDependencyNotAvailable:
														pass
							else:
														from .tokenization_plbart import PLBartTokenizer

							try:
														if not is_torch_available():
																					raise OptionalDependencyNotAvailable()
							except OptionalDependencyNotAvailable:
														pass
							else:
														from .modeling_plbart import (
														    PLBART_PRETRAINED_MODEL_ARCHIVE_LIST,
														    PLBartForCausalLM,
														    PLBartForConditionalGeneration,
														    PLBartForSequenceClassification,
														    PLBartModel,
														    PLBartPreTrainedModel,
														)


else:
							import sys

							A    =			_LazyModule(__name__, globals()['__file__'], _import_structure)



