





import json
import os
import unittest

from transformers.models.roc_bert.tokenization_roc_bert import (
    VOCAB_FILES_NAMES,
    RoCBertBasicTokenizer,
    RoCBertTokenizer,
    RoCBertWordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english
@require_tokenizers
class 			UpperCamelCase_		(  _lowerCamelCase							,  unittest.TestCase					):
					lowerCAmelCase_							=	RoCBertTokenizer
					lowerCAmelCase_							=	None
					lowerCAmelCase_							=	False
					lowerCAmelCase_							=	True
					lowerCAmelCase_							=	filter_non_english




					def 	lowerCAmelCase			(		self  )		->							int:
										super().setUp()

										_snake_case									=				['[UNK]', '[CLS]', '[SEP]', '[PAD]', '[MASK]', '你', '好', '是', '谁', 'a', 'b', 'c', 'd']
										_snake_case									=				{}
										_snake_case									=				{}
										for i, value in enumerate(lowerCAmelCase_  ):
															_snake_case									=				i
															_snake_case									=				i
										_snake_case									=				os.path.join(self.tmpdirname				,						VOCAB_FILES_NAMES['vocab_file']  )
										_snake_case									=				os.path.join(self.tmpdirname				,						VOCAB_FILES_NAMES['word_shape_file']  )
										_snake_case									=				os.path.join(self.tmpdirname				,						VOCAB_FILES_NAMES['word_pronunciation_file']  )
										with open(self.vocab_file				,						'w'				,						encoding='utf-8'  ) as vocab_writer:
															vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]  )  )
										with open(self.word_shape_file				,						'w'				,						encoding='utf-8'  ) as word_shape_writer:
															json.dump(lowerCAmelCase_				,						lowerCAmelCase_				,						ensure_ascii=lowerCAmelCase_  )
										with open(self.word_pronunciation_file				,						'w'				,						encoding='utf-8'  ) as word_pronunciation_writer:
															json.dump(lowerCAmelCase_				,						lowerCAmelCase_				,						ensure_ascii=lowerCAmelCase_  )




					def 	lowerCAmelCase			(		self  )		->							Any:
										_snake_case									=				self.tokenizer_class(self.vocab_file				,						self.word_shape_file				,						self.word_pronunciation_file  )

										_snake_case									=				tokenizer.tokenize('你好[SEP]你是谁'  )
										self.assertListEqual(lowerCAmelCase_				,						['你', '好', '[SEP]', '你', '是', '谁']  )
										self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCAmelCase_  )				,						[5, 6, 2, 5, 7, 8]  )
										self.assertListEqual(tokenizer.convert_tokens_to_shape_ids(lowerCAmelCase_  )				,						[5, 6, 2, 5, 7, 8]  )
										self.assertListEqual(tokenizer.convert_tokens_to_pronunciation_ids(lowerCAmelCase_  )				,						[5, 6, 2, 5, 7, 8]  )




					def 	lowerCAmelCase			(		self  )		->							Optional[int]:
										_snake_case									=				RoCBertBasicTokenizer()

										self.assertListEqual(tokenizer.tokenize('ah\u535A\u63A8zz'  )				,						['ah', '\u535A', '\u63A8', 'zz']  )




					def 	lowerCAmelCase			(		self  )		->							List[str]:
										_snake_case									=				RoCBertBasicTokenizer(do_lower_case=lowerCAmelCase_  )

										self.assertListEqual(
										    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '  )				,						['hello', '!', 'how', 'are', 'you', '?']  )
										self.assertListEqual(tokenizer.tokenize('H\u00E9llo'  )				,						['hello']  )




					def 	lowerCAmelCase			(		self  )		->							Optional[int]:
										_snake_case									=				RoCBertBasicTokenizer(do_lower_case=lowerCAmelCase_				,						strip_accents=lowerCAmelCase_  )

										self.assertListEqual(
										    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '  )				,						['hällo', '!', 'how', 'are', 'you', '?']  )
										self.assertListEqual(tokenizer.tokenize('H\u00E9llo'  )				,						['h\u00E9llo']  )




					def 	lowerCAmelCase			(		self  )		->							Dict:
										_snake_case									=				RoCBertBasicTokenizer(do_lower_case=lowerCAmelCase_				,						strip_accents=lowerCAmelCase_  )

										self.assertListEqual(
										    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '  )				,						['hallo', '!', 'how', 'are', 'you', '?']  )
										self.assertListEqual(tokenizer.tokenize('H\u00E9llo'  )				,						['hello']  )




					def 	lowerCAmelCase			(		self  )		->							Any:
										_snake_case									=				RoCBertBasicTokenizer(do_lower_case=lowerCAmelCase_  )

										self.assertListEqual(
										    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '  )				,						['hallo', '!', 'how', 'are', 'you', '?']  )
										self.assertListEqual(tokenizer.tokenize('H\u00E9llo'  )				,						['hello']  )




					def 	lowerCAmelCase			(		self  )		->							List[str]:
										_snake_case									=				RoCBertBasicTokenizer(do_lower_case=lowerCAmelCase_  )

										self.assertListEqual(
										    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '  )				,						['HeLLo', '!', 'how', 'Are', 'yoU', '?']  )




					def 	lowerCAmelCase			(		self  )		->							Optional[int]:
										_snake_case									=				RoCBertBasicTokenizer(do_lower_case=lowerCAmelCase_				,						strip_accents=lowerCAmelCase_  )

										self.assertListEqual(
										    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '  )				,						['HäLLo', '!', 'how', 'Are', 'yoU', '?']  )




					def 	lowerCAmelCase			(		self  )		->							Tuple:
										_snake_case									=				RoCBertBasicTokenizer(do_lower_case=lowerCAmelCase_				,						strip_accents=lowerCAmelCase_  )

										self.assertListEqual(
										    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '  )				,						['HaLLo', '!', 'how', 'Are', 'yoU', '?']  )




					def 	lowerCAmelCase			(		self  )		->							int:
										_snake_case									=				RoCBertBasicTokenizer(do_lower_case=lowerCAmelCase_				,						never_split=['[UNK]']  )

										self.assertListEqual(
										    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU? [UNK]'  )				,						['HeLLo', '!', 'how', 'Are', 'yoU', '?', '[UNK]']  )




					def 	lowerCAmelCase			(		self  )		->							Union[str, Any]:
										_snake_case									=				['[UNK]', '[CLS]', '[SEP]', 'want', '##want', '##ed', 'wa', 'un', 'runn', '##ing']

										_snake_case									=				{}
										for i, token in enumerate(lowerCAmelCase_  ):
															_snake_case									=				i
										_snake_case									=				RoCBertWordpieceTokenizer(vocab=lowerCAmelCase_				,						unk_token='[UNK]'  )

										self.assertListEqual(tokenizer.tokenize(''  )				,						[]  )

										self.assertListEqual(tokenizer.tokenize('unwanted running'  )				,						['un', '##want', '##ed', 'runn', '##ing']  )

										self.assertListEqual(tokenizer.tokenize('unwantedX running'  )				,						['[UNK]', 'runn', '##ing']  )




					def 	lowerCAmelCase			(		self  )		->							int:
										self.assertTrue(_is_whitespace(' '  )  )
										self.assertTrue(_is_whitespace('\t'  )  )
										self.assertTrue(_is_whitespace('\r'  )  )
										self.assertTrue(_is_whitespace('\n'  )  )
										self.assertTrue(_is_whitespace('\u00A0'  )  )

										self.assertFalse(_is_whitespace('A'  )  )
										self.assertFalse(_is_whitespace('-'  )  )




					def 	lowerCAmelCase			(		self  )		->							int:
										self.assertTrue(_is_control('\u0005'  )  )

										self.assertFalse(_is_control('A'  )  )
										self.assertFalse(_is_control(' '  )  )
										self.assertFalse(_is_control('\t'  )  )
										self.assertFalse(_is_control('\r'  )  )




					def 	lowerCAmelCase			(		self  )		->							List[str]:
										self.assertTrue(_is_punctuation('-'  )  )
										self.assertTrue(_is_punctuation('$'  )  )
										self.assertTrue(_is_punctuation('`'  )  )
										self.assertTrue(_is_punctuation('.'  )  )

										self.assertFalse(_is_punctuation('A'  )  )
										self.assertFalse(_is_punctuation(' '  )  )




					def 	lowerCAmelCase			(		self  )		->							Tuple:
										_snake_case									=				self.get_tokenizer()

										# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
										self.assertListEqual([tokenizer.tokenize(lowerCAmelCase_  ) for t in ['Test', '\xad', 'test']]				,						[['[UNK]'], [], ['[UNK]']]  )

										if self.test_rust_tokenizer:
															_snake_case									=				self.get_rust_tokenizer()
															self.assertListEqual(
															    [rust_tokenizer.tokenize(lowerCAmelCase_  ) for t in ['Test', '\xad', 'test']]				,						[['[UNK]'], [], ['[UNK]']]  )




					def 	lowerCAmelCase			(		self  )		->							str:
										for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
															with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''  ):
																				_snake_case									=				self.rust_tokenizer_class.from_pretrained(lowerCAmelCase_				,						**lowerCAmelCase_  )

																				_snake_case									=				F'''A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'''
																				_snake_case									=				tokenizer_r.encode_plus(
																				    lowerCAmelCase_				,						return_attention_mask=lowerCAmelCase_				,						return_token_type_ids=lowerCAmelCase_				,						return_offsets_mapping=lowerCAmelCase_				,						add_special_tokens=lowerCAmelCase_				,						)

																				_snake_case									=				tokenizer_r.do_lower_case if hasattr(lowerCAmelCase_				,						'do_lower_case'  ) else False
																				_snake_case									=				(
																				    [
																				        ((0, 0), tokenizer_r.cls_token),
																				        ((0, 1), 'A'),
																				        ((1, 2), ','),
																				        ((3, 5), 'na'),
																				        ((5, 6), '##ï'),
																				        ((6, 8), '##ve'),
																				        ((9, 15), tokenizer_r.mask_token),
																				        ((16, 21), 'Allen'),
																				        ((21, 23), '##NL'),
																				        ((23, 24), '##P'),
																				        ((25, 33), 'sentence'),
																				        ((33, 34), '.'),
																				        ((0, 0), tokenizer_r.sep_token),
																				    ]
																				    if not do_lower_case
																				    else [
																				        ((0, 0), tokenizer_r.cls_token),
																				        ((0, 1), 'a'),
																				        ((1, 2), ','),
																				        ((3, 8), 'naive'),
																				        ((9, 15), tokenizer_r.mask_token),
																				        ((16, 21), 'allen'),
																				        ((21, 23), '##nl'),
																				        ((23, 24), '##p'),
																				        ((25, 33), 'sentence'),
																				        ((33, 34), '.'),
																				        ((0, 0), tokenizer_r.sep_token),
																				    ]
																				)

																				self.assertEqual(
																				    [e[1] for e in expected_results]				,						tokenizer_r.convert_ids_to_tokens(tokens['input_ids']  )  )
																				self.assertEqual([e[0] for e in expected_results]				,						tokens['offset_mapping']  )




					def 	lowerCAmelCase			(		self  )		->							str:
										_snake_case									=				['的', '人', '有']
										_snake_case									=				''.join(lowerCAmelCase_  )
										for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
															with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''  ):
																				_snake_case									=				True
																				_snake_case									=				self.tokenizer_class.from_pretrained(lowerCAmelCase_				,						**lowerCAmelCase_  )
																				_snake_case									=				self.rust_tokenizer_class.from_pretrained(lowerCAmelCase_				,						**lowerCAmelCase_  )

																				_snake_case									=				tokenizer_p.encode(lowerCAmelCase_				,						add_special_tokens=lowerCAmelCase_  )
																				_snake_case									=				tokenizer_r.encode(lowerCAmelCase_				,						add_special_tokens=lowerCAmelCase_  )

																				_snake_case									=				tokenizer_r.convert_ids_to_tokens(lowerCAmelCase_  )
																				_snake_case									=				tokenizer_p.convert_ids_to_tokens(lowerCAmelCase_  )

																				# it is expected that each Chinese character is not preceded by "##"
																				self.assertListEqual(lowerCAmelCase_				,						lowerCAmelCase_  )
																				self.assertListEqual(lowerCAmelCase_				,						lowerCAmelCase_  )

																				_snake_case									=				False
																				_snake_case									=				self.rust_tokenizer_class.from_pretrained(lowerCAmelCase_				,						**lowerCAmelCase_  )
																				_snake_case									=				self.tokenizer_class.from_pretrained(lowerCAmelCase_				,						**lowerCAmelCase_  )

																				_snake_case									=				tokenizer_r.encode(lowerCAmelCase_				,						add_special_tokens=lowerCAmelCase_  )
																				_snake_case									=				tokenizer_p.encode(lowerCAmelCase_				,						add_special_tokens=lowerCAmelCase_  )

																				_snake_case									=				tokenizer_r.convert_ids_to_tokens(lowerCAmelCase_  )
																				_snake_case									=				tokenizer_p.convert_ids_to_tokens(lowerCAmelCase_  )

																				# it is expected that only the first Chinese character is not preceded by "##".
																				_snake_case									=				[
																				    F'''##{token}''' if idx != 0 else token for idx, token in enumerate(lowerCAmelCase_  )
																				]
																				self.assertListEqual(lowerCAmelCase_				,						lowerCAmelCase_  )
																				self.assertListEqual(lowerCAmelCase_				,						lowerCAmelCase_  )




					@slow
					def 	lowerCAmelCase			(		self  )		->							Dict:
										_snake_case									=				self.tokenizer_class(self.vocab_file				,						self.word_shape_file				,						self.word_pronunciation_file  )

										_snake_case									=				tokenizer.encode('你好'				,						add_special_tokens=lowerCAmelCase_  )
										_snake_case									=				tokenizer.encode('你是谁'				,						add_special_tokens=lowerCAmelCase_  )

										_snake_case									=				tokenizer.build_inputs_with_special_tokens(lowerCAmelCase_  )
										_snake_case									=				tokenizer.build_inputs_with_special_tokens(lowerCAmelCase_				,						lowerCAmelCase_  )

										assert encoded_sentence == [1] + text + [2]
										assert encoded_pair == [1] + text + [2] + text_a + [2]






					def 	lowerCAmelCase			(		self  )		->							str:
										_snake_case									=				self.get_tokenizers(do_lower_case=lowerCAmelCase_  )
										for tokenizer in tokenizers:
															with self.subTest(F'''{tokenizer.__class__.__name__}'''  ):
																				_snake_case									=				'你好，你是谁'
																				_snake_case									=				tokenizer.tokenize(lowerCAmelCase_  )
																				_snake_case									=				tokenizer.convert_tokens_to_ids(lowerCAmelCase_  )
																				_snake_case									=				tokenizer.convert_tokens_to_shape_ids(lowerCAmelCase_  )
																				_snake_case									=				tokenizer.convert_tokens_to_pronunciation_ids(lowerCAmelCase_  )
																				_snake_case									=				tokenizer.prepare_for_model(
																				    lowerCAmelCase_				,						lowerCAmelCase_				,						lowerCAmelCase_				,						add_special_tokens=lowerCAmelCase_  )

																				_snake_case									=				tokenizer.encode_plus(lowerCAmelCase_				,						add_special_tokens=lowerCAmelCase_  )

																				self.assertEqual(lowerCAmelCase_				,						lowerCAmelCase_  )










import copy
import tempfile
import unittest

from huggingface_hub import HfFolder, delete_repo
from parameterized import parameterized
from requests.exceptions import HTTPError

from transformers import AutoConfig, GenerationConfig
from transformers.testing_utils import TOKEN, USER, is_staging_test
class 			UpperCamelCase_		(  unittest.TestCase					):




					@parameterized.expand([(None,), ('foo.json',)]  )
					def 	lowerCAmelCase			(		self				,						lowerCAmelCase_  )		->							str:
										_snake_case									=				GenerationConfig(
										    do_sample=lowerCAmelCase_				,						temperature=0.7				,						length_penalty=1.0				,						bad_words_ids=[[1, 2, 3], [4, 5]]				,						)
										with tempfile.TemporaryDirectory() as tmp_dir:
															config.save_pretrained(lowerCAmelCase_				,						config_name=lowerCAmelCase_  )
															_snake_case									=				GenerationConfig.from_pretrained(lowerCAmelCase_				,						config_name=lowerCAmelCase_  )

										# Checks parameters that were specified
										self.assertEqual(loaded_config.do_sample				,						lowerCAmelCase_  )
										self.assertEqual(loaded_config.temperature				,						0.7  )
										self.assertEqual(loaded_config.length_penalty				,						1.0  )
										self.assertEqual(loaded_config.bad_words_ids				,						[[1, 2, 3], [4, 5]]  )

										# Checks parameters that were not specified (defaults)
										self.assertEqual(loaded_config.top_k				,						50  )
										self.assertEqual(loaded_config.max_length				,						20  )
										self.assertEqual(loaded_config.max_time				,						lowerCAmelCase_  )




					def 	lowerCAmelCase			(		self  )		->							Optional[int]:
										_snake_case									=				AutoConfig.from_pretrained('gpt2'  )
										_snake_case									=				GenerationConfig.from_model_config(lowerCAmelCase_  )
										_snake_case									=				GenerationConfig()

										# The generation config has loaded a few non-default parameters from the model config
										self.assertNotEqual(lowerCAmelCase_				,						lowerCAmelCase_  )

										# One of those parameters is eos_token_id -- check if it matches
										self.assertNotEqual(generation_config_from_model.eos_token_id				,						default_generation_config.eos_token_id  )
										self.assertEqual(generation_config_from_model.eos_token_id				,						model_config.eos_token_id  )




					def 	lowerCAmelCase			(		self  )		->							Tuple:
										_snake_case									=				GenerationConfig()
										_snake_case									=				{
										    'max_new_tokens': 1024,
										    'foo': 'bar',
										}
										_snake_case									=				copy.deepcopy(lowerCAmelCase_  )
										_snake_case									=				generation_config.update(**lowerCAmelCase_  )

										# update_kwargs was not modified (no side effects)
										self.assertEqual(lowerCAmelCase_				,						lowerCAmelCase_  )

										# update_kwargs was used to update the config on valid attributes
										self.assertEqual(generation_config.max_new_tokens				,						1024  )

										# `.update()` returns a dictionary of unused kwargs
										self.assertEqual(lowerCAmelCase_				,						{'foo': 'bar'}  )




					def 	lowerCAmelCase			(		self  )		->							Optional[int]:
										_snake_case									=				GenerationConfig()
										_snake_case									=				'bar'

										with tempfile.TemporaryDirectory('test-generation-config'  ) as tmp_dir:
															generation_config.save_pretrained(lowerCAmelCase_  )

															_snake_case									=				GenerationConfig.from_pretrained(lowerCAmelCase_  )
										# update_kwargs was used to update the config on valid attributes
										self.assertEqual(new_config.foo				,						'bar'  )

										_snake_case									=				GenerationConfig.from_model_config(lowerCAmelCase_  )
										assert not hasattr(lowerCAmelCase_				,						'foo'  )  # no new kwargs should be initialized if from config






					def 	lowerCAmelCase			(		self  )		->							List[Any]:
										_snake_case									=				GenerationConfig()
										self.assertEqual(default_config.temperature				,						1.0  )
										self.assertEqual(default_config.do_sample				,						lowerCAmelCase_  )
										self.assertEqual(default_config.num_beams				,						1  )

										_snake_case									=				GenerationConfig(
										    do_sample=lowerCAmelCase_				,						temperature=0.7				,						length_penalty=1.0				,						bad_words_ids=[[1, 2, 3], [4, 5]]				,						)
										self.assertEqual(config.temperature				,						0.7  )
										self.assertEqual(config.do_sample				,						lowerCAmelCase_  )
										self.assertEqual(config.num_beams				,						1  )

										with tempfile.TemporaryDirectory() as tmp_dir:
															config.save_pretrained(lowerCAmelCase_  )
															_snake_case									=				GenerationConfig.from_pretrained(lowerCAmelCase_				,						temperature=1.0  )

										self.assertEqual(loaded_config.temperature				,						1.0  )
										self.assertEqual(loaded_config.do_sample				,						lowerCAmelCase_  )
										self.assertEqual(loaded_config.num_beams				,						1  )  # default value





@is_staging_test
class 			UpperCamelCase_		(  unittest.TestCase					):




					@classmethod
					def 	lowerCAmelCase			(		cls  )		->							List[str]:
										_snake_case									=				TOKEN
										HfFolder.save_token(lowerCAmelCase_  )




					@classmethod
					def 	lowerCAmelCase			(		cls  )		->							Union[str, Any]:
										try:
															delete_repo(token=cls._token				,						repo_id='test-generation-config'  )
										except HTTPError:
															pass

										try:
															delete_repo(token=cls._token				,						repo_id='valid_org/test-generation-config-org'  )
										except HTTPError:
															pass




					def 	lowerCAmelCase			(		self  )		->							int:
										_snake_case									=				GenerationConfig(
										    do_sample=lowerCAmelCase_				,						temperature=0.7				,						length_penalty=1.0				,						)
										config.push_to_hub('test-generation-config'				,						use_auth_token=self._token  )

										_snake_case									=				GenerationConfig.from_pretrained(F'''{USER}/test-generation-config'''  )
										for k, v in config.to_dict().items():
															if k != "transformers_version":
																				self.assertEqual(lowerCAmelCase_				,						getattr(lowerCAmelCase_				,						lowerCAmelCase_  )  )

        # Reset repo
										delete_repo(token=self._token				,						repo_id='test-generation-config'  )

										# Push to hub via save_pretrained
										with tempfile.TemporaryDirectory() as tmp_dir:
															config.save_pretrained(
															    lowerCAmelCase_				,						repo_id='test-generation-config'				,						push_to_hub=lowerCAmelCase_				,						use_auth_token=self._token  )

										_snake_case									=				GenerationConfig.from_pretrained(F'''{USER}/test-generation-config'''  )
										for k, v in config.to_dict().items():
															if k != "transformers_version":
																				self.assertEqual(lowerCAmelCase_				,						getattr(lowerCAmelCase_				,						lowerCAmelCase_  )  )






					def 	lowerCAmelCase			(		self  )		->							List[str]:
										_snake_case									=				GenerationConfig(
										    do_sample=lowerCAmelCase_				,						temperature=0.7				,						length_penalty=1.0				,						)
										config.push_to_hub('valid_org/test-generation-config-org'				,						use_auth_token=self._token  )

										_snake_case									=				GenerationConfig.from_pretrained('valid_org/test-generation-config-org'  )
										for k, v in config.to_dict().items():
															if k != "transformers_version":
																				self.assertEqual(lowerCAmelCase_				,						getattr(lowerCAmelCase_				,						lowerCAmelCase_  )  )

        # Reset repo
										delete_repo(token=self._token				,						repo_id='valid_org/test-generation-config-org'  )

										# Push to hub via save_pretrained
										with tempfile.TemporaryDirectory() as tmp_dir:
															config.save_pretrained(
															    lowerCAmelCase_				,						repo_id='valid_org/test-generation-config-org'				,						push_to_hub=lowerCAmelCase_				,						use_auth_token=self._token  )

										_snake_case									=				GenerationConfig.from_pretrained('valid_org/test-generation-config-org'  )
										for k, v in config.to_dict().items():
															if k != "transformers_version":
																				self.assertEqual(lowerCAmelCase_				,						getattr(lowerCAmelCase_				,						lowerCAmelCase_  )  )



