"""simple docstring"""







import json
import os
import unittest

from transformers.models.roc_bert.tokenization_roc_bert import (
    VOCAB_FILES_NAMES,
    RoCBertBasicTokenizer,
    RoCBertTokenizer,
    RoCBertWordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english



@require_tokenizers
class  _lowerCamelCase			(__lowerCamelCase						,							unittest.TestCase     ):
	_snake_case     =  RoCBertTokenizer
	_snake_case     =  None
	_snake_case     =  False
	_snake_case     =  True
	_snake_case     =  filter_non_english







	def 							__UpperCAmelCase					( self    :       Optional[Any]						):

		"""simple docstring"""






		super().setUp()

		_lowercase :    List[Any]							=							['[UNK]', '[CLS]', '[SEP]', '[PAD]', '[MASK]', '你', '好', '是', '谁', 'a', 'b', 'c', 'd']
		_lowercase :    Optional[int]							=							{}
		_lowercase :    Tuple							=							{}
		for i, value in enumerate(lowerCamelCase_						):
			_lowercase :    Any							=							i
			_lowercase :    Dict							=							i
		_lowercase :    Optional[int]							=							os.path.join(self.tmpdirname			,					VOCAB_FILES_NAMES['vocab_file']						)
		_lowercase :    int							=							os.path.join(self.tmpdirname			,					VOCAB_FILES_NAMES['word_shape_file']						)
		_lowercase :    List[str]							=							os.path.join(self.tmpdirname			,					VOCAB_FILES_NAMES['word_pronunciation_file']						)
		with open(self.vocab_file			,					'w'			,					encoding='utf-8'						) as vocab_writer:
			vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]						)						)
		with open(self.word_shape_file			,					'w'			,					encoding='utf-8'						) as word_shape_writer:
			json.dump(lowerCamelCase_			,					lowerCamelCase_			,					ensure_ascii=lowerCamelCase_						)
		with open(self.word_pronunciation_file			,					'w'			,					encoding='utf-8'						) as word_pronunciation_writer:
			json.dump(lowerCamelCase_			,					lowerCamelCase_			,					ensure_ascii=lowerCamelCase_						)







	def 							__UpperCAmelCase					( self    :       Union[str, Any]						):

		"""simple docstring"""






		_lowercase :    Any							=							self.tokenizer_class(self.vocab_file			,					self.word_shape_file			,					self.word_pronunciation_file						)

		_lowercase :    Any							=							tokenizer.tokenize('你好[SEP]你是谁'						)
		self.assertListEqual(lowerCamelCase_			,					['你', '好', '[SEP]', '你', '是', '谁']						)
		self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCamelCase_						)			,					[5, 6, 2, 5, 7, 8]						)
		self.assertListEqual(tokenizer.convert_tokens_to_shape_ids(lowerCamelCase_						)			,					[5, 6, 2, 5, 7, 8]						)
		self.assertListEqual(tokenizer.convert_tokens_to_pronunciation_ids(lowerCamelCase_						)			,					[5, 6, 2, 5, 7, 8]						)







	def 							__UpperCAmelCase					( self    :       Any						):

		"""simple docstring"""






		_lowercase :    Tuple							=							RoCBertBasicTokenizer()

		self.assertListEqual(tokenizer.tokenize('ah\u535A\u63A8zz'						)			,					['ah', '\u535A', '\u63A8', 'zz']						)







	def 							__UpperCAmelCase					( self    :       str						):

		"""simple docstring"""






		_lowercase :    Optional[int]							=							RoCBertBasicTokenizer(do_lower_case=lowerCamelCase_						)

		self.assertListEqual(
		    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '						)			,					['hello', '!', 'how', 'are', 'you', '?']						)
		self.assertListEqual(tokenizer.tokenize('H\u00E9llo'						)			,					['hello']						)







	def 							__UpperCAmelCase					( self    :       Tuple						):

		"""simple docstring"""






		_lowercase :    int							=							RoCBertBasicTokenizer(do_lower_case=lowerCamelCase_			,					strip_accents=lowerCamelCase_						)

		self.assertListEqual(
		    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '						)			,					['hällo', '!', 'how', 'are', 'you', '?']						)
		self.assertListEqual(tokenizer.tokenize('H\u00E9llo'						)			,					['h\u00E9llo']						)







	def 							__UpperCAmelCase					( self    :       Dict						):

		"""simple docstring"""






		_lowercase :    List[str]							=							RoCBertBasicTokenizer(do_lower_case=lowerCamelCase_			,					strip_accents=lowerCamelCase_						)

		self.assertListEqual(
		    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '						)			,					['hallo', '!', 'how', 'are', 'you', '?']						)
		self.assertListEqual(tokenizer.tokenize('H\u00E9llo'						)			,					['hello']						)







	def 							__UpperCAmelCase					( self    :       Dict						):

		"""simple docstring"""






		_lowercase :    Optional[Any]							=							RoCBertBasicTokenizer(do_lower_case=lowerCamelCase_						)

		self.assertListEqual(
		    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '						)			,					['hallo', '!', 'how', 'are', 'you', '?']						)
		self.assertListEqual(tokenizer.tokenize('H\u00E9llo'						)			,					['hello']						)







	def 							__UpperCAmelCase					( self    :       int						):

		"""simple docstring"""






		_lowercase :    List[Any]							=							RoCBertBasicTokenizer(do_lower_case=lowerCamelCase_						)

		self.assertListEqual(
		    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '						)			,					['HeLLo', '!', 'how', 'Are', 'yoU', '?']						)







	def 							__UpperCAmelCase					( self    :       List[str]						):

		"""simple docstring"""






		_lowercase :    int							=							RoCBertBasicTokenizer(do_lower_case=lowerCamelCase_			,					strip_accents=lowerCamelCase_						)

		self.assertListEqual(
		    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '						)			,					['HäLLo', '!', 'how', 'Are', 'yoU', '?']						)







	def 							__UpperCAmelCase					( self    :       Tuple						):

		"""simple docstring"""






		_lowercase :    Dict							=							RoCBertBasicTokenizer(do_lower_case=lowerCamelCase_			,					strip_accents=lowerCamelCase_						)

		self.assertListEqual(
		    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '						)			,					['HaLLo', '!', 'how', 'Are', 'yoU', '?']						)







	def 							__UpperCAmelCase					( self    :       int						):

		"""simple docstring"""






		_lowercase :    Union[str, Any]							=							RoCBertBasicTokenizer(do_lower_case=lowerCamelCase_			,					never_split=['[UNK]']						)

		self.assertListEqual(
		    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU? [UNK]'						)			,					['HeLLo', '!', 'how', 'Are', 'yoU', '?', '[UNK]']						)







	def 							__UpperCAmelCase					( self    :       Tuple						):

		"""simple docstring"""






		_lowercase :    List[str]							=							['[UNK]', '[CLS]', '[SEP]', 'want', '##want', '##ed', 'wa', 'un', 'runn', '##ing']

		_lowercase :    List[str]							=							{}
		for i, token in enumerate(lowerCamelCase_						):
			_lowercase :    str							=							i
		_lowercase :    Tuple							=							RoCBertWordpieceTokenizer(vocab=lowerCamelCase_			,					unk_token='[UNK]'						)

		self.assertListEqual(tokenizer.tokenize(''						)			,					[]						)

		self.assertListEqual(tokenizer.tokenize('unwanted running'						)			,					['un', '##want', '##ed', 'runn', '##ing']						)

		self.assertListEqual(tokenizer.tokenize('unwantedX running'						)			,					['[UNK]', 'runn', '##ing']						)







	def 							__UpperCAmelCase					( self    :       Optional[int]						):

		"""simple docstring"""






		self.assertTrue(_is_whitespace(' '						)						)
		self.assertTrue(_is_whitespace('\t'						)						)
		self.assertTrue(_is_whitespace('\r'						)						)
		self.assertTrue(_is_whitespace('\n'						)						)
		self.assertTrue(_is_whitespace('\u00A0'						)						)

		self.assertFalse(_is_whitespace('A'						)						)
		self.assertFalse(_is_whitespace('-'						)						)







	def 							__UpperCAmelCase					( self    :       List[str]						):

		"""simple docstring"""






		self.assertTrue(_is_control('\u0005'						)						)

		self.assertFalse(_is_control('A'						)						)
		self.assertFalse(_is_control(' '						)						)
		self.assertFalse(_is_control('\t'						)						)
		self.assertFalse(_is_control('\r'						)						)







	def 							__UpperCAmelCase					( self    :       Optional[int]						):

		"""simple docstring"""






		self.assertTrue(_is_punctuation('-'						)						)
		self.assertTrue(_is_punctuation('$'						)						)
		self.assertTrue(_is_punctuation('`'						)						)
		self.assertTrue(_is_punctuation('.'						)						)

		self.assertFalse(_is_punctuation('A'						)						)
		self.assertFalse(_is_punctuation(' '						)						)







	def 							__UpperCAmelCase					( self    :       Optional[int]						):

		"""simple docstring"""






		_lowercase :    Any							=							self.get_tokenizer()

		# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
		self.assertListEqual([tokenizer.tokenize(lowerCamelCase_						) for t in ['Test', '\xad', 'test']]			,					[['[UNK]'], [], ['[UNK]']]						)

		if self.test_rust_tokenizer:
			_lowercase :    str							=							self.get_rust_tokenizer()
			self.assertListEqual(
			    [rust_tokenizer.tokenize(lowerCamelCase_						) for t in ['Test', '\xad', 'test']]			,					[['[UNK]'], [], ['[UNK]']]						)







	def 							__UpperCAmelCase					( self    :       Union[str, Any]						):

		"""simple docstring"""






		for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
			with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''						):
				_lowercase :    Union[str, Any]							=							self.rust_tokenizer_class.from_pretrained(lowerCamelCase_			,					**lowerCamelCase_						)

				_lowercase :    List[Any]							=							F'''A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'''
				_lowercase :    Union[str, Any]							=							tokenizer_r.encode_plus(
				    lowerCamelCase_			,					return_attention_mask=lowerCamelCase_			,					return_token_type_ids=lowerCamelCase_			,					return_offsets_mapping=lowerCamelCase_			,					add_special_tokens=lowerCamelCase_			,					)

				_lowercase :    Union[str, Any]							=							tokenizer_r.do_lower_case if hasattr(lowerCamelCase_			,					'do_lower_case'						) else False
				_lowercase :    Optional[Any]							=							(
				    [
				        ((0, 0), tokenizer_r.cls_token),
				        ((0, 1), 'A'),
				        ((1, 2), ','),
				        ((3, 5), 'na'),
				        ((5, 6), '##ï'),
				        ((6, 8), '##ve'),
				        ((9, 1_5), tokenizer_r.mask_token),
				        ((1_6, 2_1), 'Allen'),
				        ((2_1, 2_3), '##NL'),
				        ((2_3, 2_4), '##P'),
				        ((2_5, 3_3), 'sentence'),
				        ((3_3, 3_4), '.'),
				        ((0, 0), tokenizer_r.sep_token),
				    ]
				    if not do_lower_case
				    else [
				        ((0, 0), tokenizer_r.cls_token),
				        ((0, 1), 'a'),
				        ((1, 2), ','),
				        ((3, 8), 'naive'),
				        ((9, 1_5), tokenizer_r.mask_token),
				        ((1_6, 2_1), 'allen'),
				        ((2_1, 2_3), '##nl'),
				        ((2_3, 2_4), '##p'),
				        ((2_5, 3_3), 'sentence'),
				        ((3_3, 3_4), '.'),
				        ((0, 0), tokenizer_r.sep_token),
				    ]
				)

				self.assertEqual(
				    [e[1] for e in expected_results]			,					tokenizer_r.convert_ids_to_tokens(tokens['input_ids']						)						)
				self.assertEqual([e[0] for e in expected_results]			,					tokens['offset_mapping']						)







	def 							__UpperCAmelCase					( self    :       List[Any]						):

		"""simple docstring"""






		_lowercase :    Optional[Any]							=							['的', '人', '有']
		_lowercase :    Tuple							=							''.join(lowerCamelCase_						)
		for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
			with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''						):
				_lowercase :    Any							=							True
				_lowercase :    Tuple							=							self.tokenizer_class.from_pretrained(lowerCamelCase_			,					**lowerCamelCase_						)
				_lowercase :    Optional[int]							=							self.rust_tokenizer_class.from_pretrained(lowerCamelCase_			,					**lowerCamelCase_						)

				_lowercase :    int							=							tokenizer_p.encode(lowerCamelCase_			,					add_special_tokens=lowerCamelCase_						)
				_lowercase :    Optional[Any]							=							tokenizer_r.encode(lowerCamelCase_			,					add_special_tokens=lowerCamelCase_						)

				_lowercase :    List[Any]							=							tokenizer_r.convert_ids_to_tokens(lowerCamelCase_						)
				_lowercase :    Tuple							=							tokenizer_p.convert_ids_to_tokens(lowerCamelCase_						)

				# it is expected that each Chinese character is not preceded by "##"
				self.assertListEqual(lowerCamelCase_			,					lowerCamelCase_						)
				self.assertListEqual(lowerCamelCase_			,					lowerCamelCase_						)

				_lowercase :    Union[str, Any]							=							False
				_lowercase :    List[Any]							=							self.rust_tokenizer_class.from_pretrained(lowerCamelCase_			,					**lowerCamelCase_						)
				_lowercase :    Union[str, Any]							=							self.tokenizer_class.from_pretrained(lowerCamelCase_			,					**lowerCamelCase_						)

				_lowercase :    Union[str, Any]							=							tokenizer_r.encode(lowerCamelCase_			,					add_special_tokens=lowerCamelCase_						)
				_lowercase :    Union[str, Any]							=							tokenizer_p.encode(lowerCamelCase_			,					add_special_tokens=lowerCamelCase_						)

				_lowercase :    int							=							tokenizer_r.convert_ids_to_tokens(lowerCamelCase_						)
				_lowercase :    int							=							tokenizer_p.convert_ids_to_tokens(lowerCamelCase_						)

				# it is expected that only the first Chinese character is not preceded by "##".
				_lowercase :    Optional[int]							=							[
				    F'''##{token}''' if idx != 0 else token for idx, token in enumerate(lowerCamelCase_						)
				]
				self.assertListEqual(lowerCamelCase_			,					lowerCamelCase_						)
				self.assertListEqual(lowerCamelCase_			,					lowerCamelCase_						)







	@slow
	def 							__UpperCAmelCase					( self    :       List[Any]						):

		"""simple docstring"""






		_lowercase :    Dict							=							self.tokenizer_class(self.vocab_file			,					self.word_shape_file			,					self.word_pronunciation_file						)

		_lowercase :    Optional[int]							=							tokenizer.encode('你好'			,					add_special_tokens=lowerCamelCase_						)
		_lowercase :    Tuple							=							tokenizer.encode('你是谁'			,					add_special_tokens=lowerCamelCase_						)

		_lowercase :    Tuple							=							tokenizer.build_inputs_with_special_tokens(lowerCamelCase_						)
		_lowercase :    Union[str, Any]							=							tokenizer.build_inputs_with_special_tokens(lowerCamelCase_			,					lowerCamelCase_						)

		assert encoded_sentence == [1] + text + [2]
		assert encoded_pair == [1] + text + [2] + text_a + [2]







	def 							__UpperCAmelCase					( self    :       str						):

		"""simple docstring"""






		_lowercase :    Tuple							=							self.get_tokenizers(do_lower_case=lowerCamelCase_						)
		for tokenizer in tokenizers:
			with self.subTest(F'''{tokenizer.__class__.__name__}'''						):
				_lowercase :    Union[str, Any]							=							'你好，你是谁'
				_lowercase :    Optional[Any]							=							tokenizer.tokenize(lowerCamelCase_						)
				_lowercase :    Any							=							tokenizer.convert_tokens_to_ids(lowerCamelCase_						)
				_lowercase :    Tuple							=							tokenizer.convert_tokens_to_shape_ids(lowerCamelCase_						)
				_lowercase :    Union[str, Any]							=							tokenizer.convert_tokens_to_pronunciation_ids(lowerCamelCase_						)
				_lowercase :    Dict							=							tokenizer.prepare_for_model(
				    lowerCamelCase_			,					lowerCamelCase_			,					lowerCamelCase_			,					add_special_tokens=lowerCamelCase_						)

				_lowercase :    str							=							tokenizer.encode_plus(lowerCamelCase_			,					add_special_tokens=lowerCamelCase_						)

				self.assertEqual(lowerCamelCase_			,					lowerCamelCase_						)




"""simple docstring"""







def      __lowerCAmelCase(		__UpperCAmelCase			):






	"""simple docstring"""





	_lowercase :    str							=							[0 for i in range(len(__UpperCAmelCase			)			)]

	# initialize interval's left pointer and right pointer
	_lowercase				,      _lowercase :    str							=							0, 0

	for i in range(1     ,len(__UpperCAmelCase			)			):
		# case when current index is inside the interval
		if i <= right_pointer:
			_lowercase :    Union[str, Any]							=							min(right_pointer - i + 1     ,z_result[i - left_pointer]			)
			_lowercase :    Any							=							min_edge

		while go_next(__UpperCAmelCase     ,__UpperCAmelCase     ,__UpperCAmelCase			):
			z_result[i] += 1

		# if new index's result gives us more right interval,
		# we've to update left_pointer and right_pointer
		if i + z_result[i] - 1 > right_pointer:
			_lowercase				,      _lowercase :    Optional[int]							=							i, i + z_result[i] - 1

	return z_result


def      __lowerCAmelCase(		__UpperCAmelCase     ,__UpperCAmelCase     ,__UpperCAmelCase			):






	"""simple docstring"""





	return i + z_result[i] < len(__UpperCAmelCase			) and s[z_result[i]] == s[i + z_result[i]]


def      __lowerCAmelCase(		__UpperCAmelCase     ,__UpperCAmelCase			):






	"""simple docstring"""





	_lowercase :    Union[str, Any]							=							0
	# concatenate 'pattern' and 'input_str' and call z_function
	# with concatenated string
	_lowercase :    List[str]							=							z_function(pattern + input_str			)

	for val in z_result:
		# if value is greater then length of the pattern string
		# that means this index is starting position of substring
		# which is equal to pattern string
		if val >= len(__UpperCAmelCase			):
			answer += 1

	return answer


if __name__ == "__main__":
						import doctest

						doctest.testmod()



