


'''simple docstring'''
import json
import os
import unittest

from transformers.models.roc_bert.tokenization_roc_bert import (
    VOCAB_FILES_NAMES,
    RoCBertBasicTokenizer,
    RoCBertTokenizer,
    RoCBertWordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english
@require_tokenizers
class   lowercase      (				A__      ,						unittest.TestCase   ):

		"""simple docstring"""

		_a       					=							RoCBertTokenizer
		_a       					=							None
		_a       					=							False
		_a       					=							True
		_a       					=							filter_non_english




		def        lowerCAmelCase__							(						self				):

							'''simple docstring'''


							super().setUp()

							UpperCamelCase__  :str				   =					['''[UNK]''', '''[CLS]''', '''[SEP]''', '''[PAD]''', '''[MASK]''', '''你''', '''好''', '''是''', '''谁''', '''a''', '''b''', '''c''', '''d''']
							UpperCamelCase__  :str				   =					{}
							UpperCamelCase__  :Tuple				   =					{}
							for i, value in enumerate(UpperCamelCase_				):
												UpperCamelCase__  :Optional[Any]				   =					i
												UpperCamelCase__  :Tuple				   =					i
							UpperCamelCase__  :Tuple				   =					os.path.join(self.tmpdirname       ,  VOCAB_FILES_NAMES['''vocab_file''']				)
							UpperCamelCase__  :List[str]				   =					os.path.join(self.tmpdirname       ,  VOCAB_FILES_NAMES['''word_shape_file''']				)
							UpperCamelCase__  :Any				   =					os.path.join(self.tmpdirname       ,  VOCAB_FILES_NAMES['''word_pronunciation_file''']				)
							with open(self.vocab_file       ,  '''w'''       ,  encoding='''utf-8'''				) as vocab_writer:
												vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]				)				)
							with open(self.word_shape_file       ,  '''w'''       ,  encoding='''utf-8'''				) as word_shape_writer:
												json.dump(UpperCamelCase_       ,  UpperCamelCase_       ,  ensure_ascii=UpperCamelCase_				)
							with open(self.word_pronunciation_file       ,  '''w'''       ,  encoding='''utf-8'''				) as word_pronunciation_writer:
												json.dump(UpperCamelCase_       ,  UpperCamelCase_       ,  ensure_ascii=UpperCamelCase_				)




		def        lowerCAmelCase__							(						self				):

							'''simple docstring'''


							UpperCamelCase__  :Tuple				   =					self.tokenizer_class(self.vocab_file       ,  self.word_shape_file       ,  self.word_pronunciation_file				)

							UpperCamelCase__  :Any				   =					tokenizer.tokenize('''你好[SEP]你是谁'''				)
							self.assertListEqual(UpperCamelCase_       ,  ['''你''', '''好''', '''[SEP]''', '''你''', '''是''', '''谁''']				)
							self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCamelCase_				)       ,  [5, 6, 2, 5, 7, 8]				)
							self.assertListEqual(tokenizer.convert_tokens_to_shape_ids(UpperCamelCase_				)       ,  [5, 6, 2, 5, 7, 8]				)
							self.assertListEqual(tokenizer.convert_tokens_to_pronunciation_ids(UpperCamelCase_				)       ,  [5, 6, 2, 5, 7, 8]				)




		def        lowerCAmelCase__							(						self				):

							'''simple docstring'''


							UpperCamelCase__  :Optional[int]				   =					RoCBertBasicTokenizer()

							self.assertListEqual(tokenizer.tokenize('''ah\u535A\u63A8zz'''				)       ,  ['''ah''', '''\u535A''', '''\u63A8''', '''zz''']				)




		def        lowerCAmelCase__							(						self				):

							'''simple docstring'''


							UpperCamelCase__  :Any				   =					RoCBertBasicTokenizer(do_lower_case=UpperCamelCase_				)

							self.assertListEqual(
							    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''				)       ,  ['''hello''', '''!''', '''how''', '''are''', '''you''', '''?''']				)
							self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''				)       ,  ['''hello''']				)




		def        lowerCAmelCase__							(						self				):

							'''simple docstring'''


							UpperCamelCase__  :List[Any]				   =					RoCBertBasicTokenizer(do_lower_case=UpperCamelCase_       ,  strip_accents=UpperCamelCase_				)

							self.assertListEqual(
							    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''				)       ,  ['''hällo''', '''!''', '''how''', '''are''', '''you''', '''?''']				)
							self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''				)       ,  ['''h\u00E9llo''']				)




		def        lowerCAmelCase__							(						self				):

							'''simple docstring'''


							UpperCamelCase__  :Optional[Any]				   =					RoCBertBasicTokenizer(do_lower_case=UpperCamelCase_       ,  strip_accents=UpperCamelCase_				)

							self.assertListEqual(
							    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''				)       ,  ['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']				)
							self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''				)       ,  ['''hello''']				)




		def        lowerCAmelCase__							(						self				):

							'''simple docstring'''


							UpperCamelCase__  :int				   =					RoCBertBasicTokenizer(do_lower_case=UpperCamelCase_				)

							self.assertListEqual(
							    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''				)       ,  ['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']				)
							self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''				)       ,  ['''hello''']				)




		def        lowerCAmelCase__							(						self				):

							'''simple docstring'''


							UpperCamelCase__  :List[Any]				   =					RoCBertBasicTokenizer(do_lower_case=UpperCamelCase_				)

							self.assertListEqual(
							    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''				)       ,  ['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']				)




		def        lowerCAmelCase__							(						self				):

							'''simple docstring'''


							UpperCamelCase__  :Optional[Any]				   =					RoCBertBasicTokenizer(do_lower_case=UpperCamelCase_       ,  strip_accents=UpperCamelCase_				)

							self.assertListEqual(
							    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''				)       ,  ['''HäLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']				)




		def        lowerCAmelCase__							(						self				):

							'''simple docstring'''


							UpperCamelCase__  :Any				   =					RoCBertBasicTokenizer(do_lower_case=UpperCamelCase_       ,  strip_accents=UpperCamelCase_				)

							self.assertListEqual(
							    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''				)       ,  ['''HaLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']				)




		def        lowerCAmelCase__							(						self				):

							'''simple docstring'''


							UpperCamelCase__  :Optional[int]				   =					RoCBertBasicTokenizer(do_lower_case=UpperCamelCase_       ,  never_split=['''[UNK]''']				)

							self.assertListEqual(
							    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU? [UNK]'''				)       ,  ['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''', '''[UNK]''']				)




		def        lowerCAmelCase__							(						self				):

							'''simple docstring'''


							UpperCamelCase__  :int				   =					['''[UNK]''', '''[CLS]''', '''[SEP]''', '''want''', '''##want''', '''##ed''', '''wa''', '''un''', '''runn''', '''##ing''']

							UpperCamelCase__  :Union[str, Any]				   =					{}
							for i, token in enumerate(UpperCamelCase_				):
												UpperCamelCase__  :List[Any]				   =					i
							UpperCamelCase__  :str				   =					RoCBertWordpieceTokenizer(vocab=UpperCamelCase_       ,  unk_token='''[UNK]'''				)

							self.assertListEqual(tokenizer.tokenize(''''''				)       ,  []				)

							self.assertListEqual(tokenizer.tokenize('''unwanted running'''				)       ,  ['''un''', '''##want''', '''##ed''', '''runn''', '''##ing''']				)

							self.assertListEqual(tokenizer.tokenize('''unwantedX running'''				)       ,  ['''[UNK]''', '''runn''', '''##ing''']				)




		def        lowerCAmelCase__							(						self				):

							'''simple docstring'''


							self.assertTrue(_is_whitespace(''' '''				)				)
							self.assertTrue(_is_whitespace('''\t'''				)				)
							self.assertTrue(_is_whitespace('''\r'''				)				)
							self.assertTrue(_is_whitespace('''\n'''				)				)
							self.assertTrue(_is_whitespace('''\u00A0'''				)				)

							self.assertFalse(_is_whitespace('''A'''				)				)
							self.assertFalse(_is_whitespace('''-'''				)				)




		def        lowerCAmelCase__							(						self				):

							'''simple docstring'''


							self.assertTrue(_is_control('''\u0005'''				)				)

							self.assertFalse(_is_control('''A'''				)				)
							self.assertFalse(_is_control(''' '''				)				)
							self.assertFalse(_is_control('''\t'''				)				)
							self.assertFalse(_is_control('''\r'''				)				)




		def        lowerCAmelCase__							(						self				):

							'''simple docstring'''


							self.assertTrue(_is_punctuation('''-'''				)				)
							self.assertTrue(_is_punctuation('''$'''				)				)
							self.assertTrue(_is_punctuation('''`'''				)				)
							self.assertTrue(_is_punctuation('''.'''				)				)

							self.assertFalse(_is_punctuation('''A'''				)				)
							self.assertFalse(_is_punctuation(''' '''				)				)




		def        lowerCAmelCase__							(						self				):

							'''simple docstring'''


							UpperCamelCase__  :Dict				   =					self.get_tokenizer()

							# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
							self.assertListEqual([tokenizer.tokenize(UpperCamelCase_				) for t in ['''Test''', '''\xad''', '''test''']]       ,  [['''[UNK]'''], [], ['''[UNK]''']]				)

							if self.test_rust_tokenizer:
												UpperCamelCase__  :Dict				   =					self.get_rust_tokenizer()
												self.assertListEqual(
												    [rust_tokenizer.tokenize(UpperCamelCase_				) for t in ['''Test''', '''\xad''', '''test''']]       ,  [['''[UNK]'''], [], ['''[UNK]''']]				)




		def        lowerCAmelCase__							(						self				):

							'''simple docstring'''


							for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
												with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''				):
																	UpperCamelCase__  :int				   =					self.rust_tokenizer_class.from_pretrained(UpperCamelCase_       ,  **UpperCamelCase_				)

																	UpperCamelCase__  :Optional[int]				   =					F'''A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'''
																	UpperCamelCase__  :List[str]				   =					tokenizer_r.encode_plus(
																	    UpperCamelCase_       ,  return_attention_mask=UpperCamelCase_       ,  return_token_type_ids=UpperCamelCase_       ,  return_offsets_mapping=UpperCamelCase_       ,  add_special_tokens=UpperCamelCase_       ,  )

																	UpperCamelCase__  :Tuple				   =					tokenizer_r.do_lower_case if hasattr(UpperCamelCase_       ,  '''do_lower_case'''				) else False
																	UpperCamelCase__  :Optional[int]				   =					(
																	    [
																	        ((0, 0), tokenizer_r.cls_token),
																	        ((0, 1), '''A'''),
																	        ((1, 2), ''','''),
																	        ((3, 5), '''na'''),
																	        ((5, 6), '''##ï'''),
																	        ((6, 8), '''##ve'''),
																	        ((9, 15), tokenizer_r.mask_token),
																	        ((16, 21), '''Allen'''),
																	        ((21, 23), '''##NL'''),
																	        ((23, 24), '''##P'''),
																	        ((25, 33), '''sentence'''),
																	        ((33, 34), '''.'''),
																	        ((0, 0), tokenizer_r.sep_token),
																	    ]
																	    if not do_lower_case
																	    else [
																	        ((0, 0), tokenizer_r.cls_token),
																	        ((0, 1), '''a'''),
																	        ((1, 2), ''','''),
																	        ((3, 8), '''naive'''),
																	        ((9, 15), tokenizer_r.mask_token),
																	        ((16, 21), '''allen'''),
																	        ((21, 23), '''##nl'''),
																	        ((23, 24), '''##p'''),
																	        ((25, 33), '''sentence'''),
																	        ((33, 34), '''.'''),
																	        ((0, 0), tokenizer_r.sep_token),
																	    ]
																	)

																	self.assertEqual(
																	    [e[1] for e in expected_results]       ,  tokenizer_r.convert_ids_to_tokens(tokens['''input_ids''']				)				)
																	self.assertEqual([e[0] for e in expected_results]       ,  tokens['''offset_mapping''']				)




		def        lowerCAmelCase__							(						self				):

							'''simple docstring'''


							UpperCamelCase__  :Tuple				   =					['''的''', '''人''', '''有''']
							UpperCamelCase__  :int				   =					''''''.join(UpperCamelCase_				)
							for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
												with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''				):
																	UpperCamelCase__  :List[str]				   =					True
																	UpperCamelCase__  :Tuple				   =					self.tokenizer_class.from_pretrained(UpperCamelCase_       ,  **UpperCamelCase_				)
																	UpperCamelCase__  :int				   =					self.rust_tokenizer_class.from_pretrained(UpperCamelCase_       ,  **UpperCamelCase_				)

																	UpperCamelCase__  :str				   =					tokenizer_p.encode(UpperCamelCase_       ,  add_special_tokens=UpperCamelCase_				)
																	UpperCamelCase__  :int				   =					tokenizer_r.encode(UpperCamelCase_       ,  add_special_tokens=UpperCamelCase_				)

																	UpperCamelCase__  :int				   =					tokenizer_r.convert_ids_to_tokens(UpperCamelCase_				)
																	UpperCamelCase__  :List[str]				   =					tokenizer_p.convert_ids_to_tokens(UpperCamelCase_				)

																	# it is expected that each Chinese character is not preceded by "##"
																	self.assertListEqual(UpperCamelCase_       ,  UpperCamelCase_				)
																	self.assertListEqual(UpperCamelCase_       ,  UpperCamelCase_				)

																	UpperCamelCase__  :int				   =					False
																	UpperCamelCase__  :List[Any]				   =					self.rust_tokenizer_class.from_pretrained(UpperCamelCase_       ,  **UpperCamelCase_				)
																	UpperCamelCase__  :str				   =					self.tokenizer_class.from_pretrained(UpperCamelCase_       ,  **UpperCamelCase_				)

																	UpperCamelCase__  :Optional[int]				   =					tokenizer_r.encode(UpperCamelCase_       ,  add_special_tokens=UpperCamelCase_				)
																	UpperCamelCase__  :int				   =					tokenizer_p.encode(UpperCamelCase_       ,  add_special_tokens=UpperCamelCase_				)

																	UpperCamelCase__  :Dict				   =					tokenizer_r.convert_ids_to_tokens(UpperCamelCase_				)
																	UpperCamelCase__  :int				   =					tokenizer_p.convert_ids_to_tokens(UpperCamelCase_				)

																	# it is expected that only the first Chinese character is not preceded by "##".
																	UpperCamelCase__  :int				   =					[
																	    F'''##{token}''' if idx != 0 else token for idx, token in enumerate(UpperCamelCase_				)
																	]
																	self.assertListEqual(UpperCamelCase_       ,  UpperCamelCase_				)
																	self.assertListEqual(UpperCamelCase_       ,  UpperCamelCase_				)




		@slow
		def        lowerCAmelCase__							(						self				):

							'''simple docstring'''


							UpperCamelCase__  :Union[str, Any]				   =					self.tokenizer_class(self.vocab_file       ,  self.word_shape_file       ,  self.word_pronunciation_file				)

							UpperCamelCase__  :int				   =					tokenizer.encode('''你好'''       ,  add_special_tokens=UpperCamelCase_				)
							UpperCamelCase__  :Any				   =					tokenizer.encode('''你是谁'''       ,  add_special_tokens=UpperCamelCase_				)

							UpperCamelCase__  :Dict				   =					tokenizer.build_inputs_with_special_tokens(UpperCamelCase_				)
							UpperCamelCase__  :str				   =					tokenizer.build_inputs_with_special_tokens(UpperCamelCase_       ,  UpperCamelCase_				)

							assert encoded_sentence == [1] + text + [2]
							assert encoded_pair == [1] + text + [2] + text_a + [2]




		def        lowerCAmelCase__							(						self				):

							'''simple docstring'''


							UpperCamelCase__  :Union[str, Any]				   =					self.get_tokenizers(do_lower_case=UpperCamelCase_				)
							for tokenizer in tokenizers:
												with self.subTest(F'''{tokenizer.__class__.__name__}'''				):
																	UpperCamelCase__  :Any				   =					'''你好，你是谁'''
																	UpperCamelCase__  :Optional[Any]				   =					tokenizer.tokenize(UpperCamelCase_				)
																	UpperCamelCase__  :Any				   =					tokenizer.convert_tokens_to_ids(UpperCamelCase_				)
																	UpperCamelCase__  :Optional[Any]				   =					tokenizer.convert_tokens_to_shape_ids(UpperCamelCase_				)
																	UpperCamelCase__  :List[str]				   =					tokenizer.convert_tokens_to_pronunciation_ids(UpperCamelCase_				)
																	UpperCamelCase__  :Tuple				   =					tokenizer.prepare_for_model(
																	    UpperCamelCase_       ,  UpperCamelCase_       ,  UpperCamelCase_       ,  add_special_tokens=UpperCamelCase_				)

																	UpperCamelCase__  :str				   =					tokenizer.encode_plus(UpperCamelCase_       ,  add_special_tokens=UpperCamelCase_				)

																	self.assertEqual(UpperCamelCase_       ,  UpperCamelCase_				)



'''simple docstring'''
import inspect
import os
import unittest

import torch

import accelerate
from accelerate import debug_launcher
from accelerate.test_utils import (
    execute_subprocess_async,
    require_cpu,
    require_huggingface_suite,
    require_multi_gpu,
    require_single_gpu,
)
from accelerate.utils import patch_environment
@require_huggingface_suite
class   lowercase      (				unittest.TestCase   ):

		"""simple docstring"""

		def        lowerCAmelCase__							(						self				):

							'''simple docstring'''


							UpperCamelCase__  :Any				   =					inspect.getfile(accelerate.test_utils				)
							UpperCamelCase__  :Any				   =					os.path.sep.join(
							    mod_file.split(os.path.sep				)[:-1] + ['''scripts''', '''external_deps''', '''test_metrics.py''']				)

							from accelerate.test_utils.scripts.external_deps import test_metrics  # noqa: F401

							UpperCamelCase__  :List[str]				   =					test_metrics




		@require_cpu
		def        lowerCAmelCase__							(						self				):

							'''simple docstring'''


							debug_launcher(self.test_metrics.main       ,  num_processes=1				)




		@require_cpu
		def        lowerCAmelCase__							(						self				):

							'''simple docstring'''


							debug_launcher(self.test_metrics.main				)




		@require_single_gpu
		def        lowerCAmelCase__							(						self				):

							'''simple docstring'''


							self.test_metrics.main()




		@require_multi_gpu
		def        lowerCAmelCase__							(						self				):

							'''simple docstring'''


							print(F'''Found {torch.cuda.device_count()} devices.'''				)
							UpperCamelCase__  :Optional[Any]				   =					['''torchrun''', F'''--nproc_per_node={torch.cuda.device_count()}''', self.test_file_path]
							with patch_environment(omp_num_threads=1				):
												execute_subprocess_async(UpperCamelCase_       ,  env=os.environ.copy()				)