





"""simple docstring"""





import json
import os
import unittest

from transformers.models.roc_bert.tokenization_roc_bert import (
    VOCAB_FILES_NAMES,
    RoCBertBasicTokenizer,
    RoCBertTokenizer,
    RoCBertWordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english
@require_tokenizers
class 		SCREAMING_SNAKE_CASE_				(     snake_case__							,	unittest.TestCase ):
				"""simple docstring"""
				__snake_case :							Optional[Any]									=		RoCBertTokenizer
				__snake_case :							Union[str, Any]									=		None
				__snake_case :							Optional[Any]									=		False
				__snake_case :							Union[str, Any]									=		True
				__snake_case :							Optional[int]									=		filter_non_english




				def        __lowercase	(  self	:Dict      ):
											super().setUp()

											__lowerCamelCase   :   int       =['''[UNK]''', '''[CLS]''', '''[SEP]''', '''[PAD]''', '''[MASK]''', '''你''', '''好''', '''是''', '''谁''', '''a''', '''b''', '''c''', '''d''']
											__lowerCamelCase   :   Tuple       ={}
											__lowerCamelCase   :   Union[str, Any]       ={}
											for i, value in enumerate(__lowercase      ):
																		__lowerCamelCase   :   Dict       =i
																		__lowerCamelCase   :   Optional[Any]       =i
											__lowerCamelCase   :   Union[str, Any]       =os.path.join(self.tmpdirname    ,     VOCAB_FILES_NAMES['''vocab_file''']      )
											__lowerCamelCase   :   List[str]       =os.path.join(self.tmpdirname    ,     VOCAB_FILES_NAMES['''word_shape_file''']      )
											__lowerCamelCase   :   List[str]       =os.path.join(self.tmpdirname    ,     VOCAB_FILES_NAMES['''word_pronunciation_file''']      )
											with open(self.vocab_file    ,     '''w'''    ,     encoding='''utf-8'''      ) as vocab_writer:
																		vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]      )      )
											with open(self.word_shape_file    ,     '''w'''    ,     encoding='''utf-8'''      ) as word_shape_writer:
																		json.dump(__lowercase    ,     __lowercase    ,     ensure_ascii=__lowercase      )
											with open(self.word_pronunciation_file    ,     '''w'''    ,     encoding='''utf-8'''      ) as word_pronunciation_writer:
																		json.dump(__lowercase    ,     __lowercase    ,     ensure_ascii=__lowercase      )




				def        __lowercase	(  self	:str      ):
											__lowerCamelCase   :   Optional[int]       =self.tokenizer_class(self.vocab_file    ,     self.word_shape_file    ,     self.word_pronunciation_file      )

											__lowerCamelCase   :   Optional[Any]       =tokenizer.tokenize('''你好[SEP]你是谁'''      )
											self.assertListEqual(__lowercase    ,     ['''你''', '''好''', '''[SEP]''', '''你''', '''是''', '''谁''']      )
											self.assertListEqual(tokenizer.convert_tokens_to_ids(__lowercase      )    ,     [5, 6, 2, 5, 7, 8]      )
											self.assertListEqual(tokenizer.convert_tokens_to_shape_ids(__lowercase      )    ,     [5, 6, 2, 5, 7, 8]      )
											self.assertListEqual(tokenizer.convert_tokens_to_pronunciation_ids(__lowercase      )    ,     [5, 6, 2, 5, 7, 8]      )




				def        __lowercase	(  self	:Dict      ):
											__lowerCamelCase   :   Optional[int]       =RoCBertBasicTokenizer()

											self.assertListEqual(tokenizer.tokenize('''ah\u535A\u63A8zz'''      )    ,     ['''ah''', '''\u535A''', '''\u63A8''', '''zz''']      )




				def        __lowercase	(  self	:List[Any]      ):
											__lowerCamelCase   :   Optional[Any]       =RoCBertBasicTokenizer(do_lower_case=__lowercase      )

											self.assertListEqual(
											    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''      )    ,     ['''hello''', '''!''', '''how''', '''are''', '''you''', '''?''']      )
											self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''      )    ,     ['''hello''']      )




				def        __lowercase	(  self	:List[Any]      ):
											__lowerCamelCase   :   Optional[int]       =RoCBertBasicTokenizer(do_lower_case=__lowercase    ,     strip_accents=__lowercase      )

											self.assertListEqual(
											    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''      )    ,     ['''hällo''', '''!''', '''how''', '''are''', '''you''', '''?''']      )
											self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''      )    ,     ['''h\u00E9llo''']      )




				def        __lowercase	(  self	:Dict      ):
											__lowerCamelCase   :   List[str]       =RoCBertBasicTokenizer(do_lower_case=__lowercase    ,     strip_accents=__lowercase      )

											self.assertListEqual(
											    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''      )    ,     ['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']      )
											self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''      )    ,     ['''hello''']      )




				def        __lowercase	(  self	:int      ):
											__lowerCamelCase   :   Dict       =RoCBertBasicTokenizer(do_lower_case=__lowercase      )

											self.assertListEqual(
											    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''      )    ,     ['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']      )
											self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''      )    ,     ['''hello''']      )




				def        __lowercase	(  self	:List[str]      ):
											__lowerCamelCase   :   Tuple       =RoCBertBasicTokenizer(do_lower_case=__lowercase      )

											self.assertListEqual(
											    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''      )    ,     ['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']      )




				def        __lowercase	(  self	:Tuple      ):
											__lowerCamelCase   :   str       =RoCBertBasicTokenizer(do_lower_case=__lowercase    ,     strip_accents=__lowercase      )

											self.assertListEqual(
											    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''      )    ,     ['''HäLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']      )




				def        __lowercase	(  self	:Union[str, Any]      ):
											__lowerCamelCase   :   int       =RoCBertBasicTokenizer(do_lower_case=__lowercase    ,     strip_accents=__lowercase      )

											self.assertListEqual(
											    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''      )    ,     ['''HaLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']      )




				def        __lowercase	(  self	:Union[str, Any]      ):
											__lowerCamelCase   :   Any       =RoCBertBasicTokenizer(do_lower_case=__lowercase    ,     never_split=['''[UNK]''']      )

											self.assertListEqual(
											    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU? [UNK]'''      )    ,     ['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''', '''[UNK]''']      )




				def        __lowercase	(  self	:Optional[int]      ):
											__lowerCamelCase   :   str       =['''[UNK]''', '''[CLS]''', '''[SEP]''', '''want''', '''##want''', '''##ed''', '''wa''', '''un''', '''runn''', '''##ing''']

											__lowerCamelCase   :   int       ={}
											for i, token in enumerate(__lowercase      ):
																		__lowerCamelCase   :   Optional[int]       =i
											__lowerCamelCase   :   Any       =RoCBertWordpieceTokenizer(vocab=__lowercase    ,     unk_token='''[UNK]'''      )

											self.assertListEqual(tokenizer.tokenize(''''''      )    ,     []      )

											self.assertListEqual(tokenizer.tokenize('''unwanted running'''      )    ,     ['''un''', '''##want''', '''##ed''', '''runn''', '''##ing''']      )

											self.assertListEqual(tokenizer.tokenize('''unwantedX running'''      )    ,     ['''[UNK]''', '''runn''', '''##ing''']      )




				def        __lowercase	(  self	:str      ):
											self.assertTrue(_is_whitespace(''' '''      )      )
											self.assertTrue(_is_whitespace('''\t'''      )      )
											self.assertTrue(_is_whitespace('''\r'''      )      )
											self.assertTrue(_is_whitespace('''\n'''      )      )
											self.assertTrue(_is_whitespace('''\u00A0'''      )      )

											self.assertFalse(_is_whitespace('''A'''      )      )
											self.assertFalse(_is_whitespace('''-'''      )      )




				def        __lowercase	(  self	:Union[str, Any]      ):
											self.assertTrue(_is_control('''\u0005'''      )      )

											self.assertFalse(_is_control('''A'''      )      )
											self.assertFalse(_is_control(''' '''      )      )
											self.assertFalse(_is_control('''\t'''      )      )
											self.assertFalse(_is_control('''\r'''      )      )




				def        __lowercase	(  self	:Dict      ):
											self.assertTrue(_is_punctuation('''-'''      )      )
											self.assertTrue(_is_punctuation('''$'''      )      )
											self.assertTrue(_is_punctuation('''`'''      )      )
											self.assertTrue(_is_punctuation('''.'''      )      )

											self.assertFalse(_is_punctuation('''A'''      )      )
											self.assertFalse(_is_punctuation(''' '''      )      )




				def        __lowercase	(  self	:Dict      ):
											__lowerCamelCase   :   Optional[int]       =self.get_tokenizer()

											# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
											self.assertListEqual([tokenizer.tokenize(__lowercase      ) for t in ['''Test''', '''\xad''', '''test''']]    ,     [['''[UNK]'''], [], ['''[UNK]''']]      )

											if self.test_rust_tokenizer:
																		__lowerCamelCase   :   Tuple       =self.get_rust_tokenizer()
																		self.assertListEqual(
																		    [rust_tokenizer.tokenize(__lowercase      ) for t in ['''Test''', '''\xad''', '''test''']]    ,     [['''[UNK]'''], [], ['''[UNK]''']]      )




				def        __lowercase	(  self	:Optional[Any]      ):
											for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																		with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'      ):
																									__lowerCamelCase   :   Optional[int]       =self.rust_tokenizer_class.from_pretrained(__lowercase    ,     **__lowercase      )

																									__lowerCamelCase   :   str       =f'A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'
																									__lowerCamelCase   :   int       =tokenizer_r.encode_plus(
																									    __lowercase    ,     return_attention_mask=__lowercase    ,     return_token_type_ids=__lowercase    ,     return_offsets_mapping=__lowercase    ,     add_special_tokens=__lowercase    ,     )

																									__lowerCamelCase   :   Optional[Any]       =tokenizer_r.do_lower_case if hasattr(__lowercase    ,     '''do_lower_case'''      ) else False
																									__lowerCamelCase   :   Union[str, Any]       =(
																									    [
																									        ((0, 0), tokenizer_r.cls_token),
																									        ((0, 1), '''A'''),
																									        ((1, 2), ''','''),
																									        ((3, 5), '''na'''),
																									        ((5, 6), '''##ï'''),
																									        ((6, 8), '''##ve'''),
																									        ((9, 15), tokenizer_r.mask_token),
																									        ((16, 21), '''Allen'''),
																									        ((21, 23), '''##NL'''),
																									        ((23, 24), '''##P'''),
																									        ((25, 33), '''sentence'''),
																									        ((33, 34), '''.'''),
																									        ((0, 0), tokenizer_r.sep_token),
																									    ]
																									    if not do_lower_case
																									    else [
																									        ((0, 0), tokenizer_r.cls_token),
																									        ((0, 1), '''a'''),
																									        ((1, 2), ''','''),
																									        ((3, 8), '''naive'''),
																									        ((9, 15), tokenizer_r.mask_token),
																									        ((16, 21), '''allen'''),
																									        ((21, 23), '''##nl'''),
																									        ((23, 24), '''##p'''),
																									        ((25, 33), '''sentence'''),
																									        ((33, 34), '''.'''),
																									        ((0, 0), tokenizer_r.sep_token),
																									    ]
																									)

																									self.assertEqual(
																									    [e[1] for e in expected_results]    ,     tokenizer_r.convert_ids_to_tokens(tokens['''input_ids''']      )      )
																									self.assertEqual([e[0] for e in expected_results]    ,     tokens['''offset_mapping''']      )




				def        __lowercase	(  self	:Union[str, Any]      ):
											__lowerCamelCase   :   Union[str, Any]       =['''的''', '''人''', '''有''']
											__lowerCamelCase   :   int       =''''''.join(__lowercase      )
											for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																		with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'      ):
																									__lowerCamelCase   :   Tuple       =True
																									__lowerCamelCase   :   Union[str, Any]       =self.tokenizer_class.from_pretrained(__lowercase    ,     **__lowercase      )
																									__lowerCamelCase   :   Optional[int]       =self.rust_tokenizer_class.from_pretrained(__lowercase    ,     **__lowercase      )

																									__lowerCamelCase   :   List[str]       =tokenizer_p.encode(__lowercase    ,     add_special_tokens=__lowercase      )
																									__lowerCamelCase   :   Any       =tokenizer_r.encode(__lowercase    ,     add_special_tokens=__lowercase      )

																									__lowerCamelCase   :   List[str]       =tokenizer_r.convert_ids_to_tokens(__lowercase      )
																									__lowerCamelCase   :   Optional[Any]       =tokenizer_p.convert_ids_to_tokens(__lowercase      )

																									# it is expected that each Chinese character is not preceded by "##"
																									self.assertListEqual(__lowercase    ,     __lowercase      )
																									self.assertListEqual(__lowercase    ,     __lowercase      )

																									__lowerCamelCase   :   Optional[int]       =False
																									__lowerCamelCase   :   str       =self.rust_tokenizer_class.from_pretrained(__lowercase    ,     **__lowercase      )
																									__lowerCamelCase   :   Optional[Any]       =self.tokenizer_class.from_pretrained(__lowercase    ,     **__lowercase      )

																									__lowerCamelCase   :   Optional[int]       =tokenizer_r.encode(__lowercase    ,     add_special_tokens=__lowercase      )
																									__lowerCamelCase   :   int       =tokenizer_p.encode(__lowercase    ,     add_special_tokens=__lowercase      )

																									__lowerCamelCase   :   List[Any]       =tokenizer_r.convert_ids_to_tokens(__lowercase      )
																									__lowerCamelCase   :   Any       =tokenizer_p.convert_ids_to_tokens(__lowercase      )

																									# it is expected that only the first Chinese character is not preceded by "##".
																									__lowerCamelCase   :   Dict       =[
																									    f'##{token}' if idx != 0 else token for idx, token in enumerate(__lowercase      )
																									]
																									self.assertListEqual(__lowercase    ,     __lowercase      )
																									self.assertListEqual(__lowercase    ,     __lowercase      )




				@slow
				def        __lowercase	(  self	:str      ):
											__lowerCamelCase   :   Optional[Any]       =self.tokenizer_class(self.vocab_file    ,     self.word_shape_file    ,     self.word_pronunciation_file      )

											__lowerCamelCase   :   Any       =tokenizer.encode('''你好'''    ,     add_special_tokens=__lowercase      )
											__lowerCamelCase   :   Tuple       =tokenizer.encode('''你是谁'''    ,     add_special_tokens=__lowercase      )

											__lowerCamelCase   :   Optional[int]       =tokenizer.build_inputs_with_special_tokens(__lowercase      )
											__lowerCamelCase   :   Optional[Any]       =tokenizer.build_inputs_with_special_tokens(__lowercase    ,     __lowercase      )

											assert encoded_sentence == [1] + text + [2]
											assert encoded_pair == [1] + text + [2] + text_a + [2]




				def        __lowercase	(  self	:Tuple      ):
											__lowerCamelCase   :   Dict       =self.get_tokenizers(do_lower_case=__lowercase      )
											for tokenizer in tokenizers:
																		with self.subTest(f'{tokenizer.__class__.__name__}'      ):
																									__lowerCamelCase   :   Optional[Any]       ='''你好，你是谁'''
																									__lowerCamelCase   :   Optional[int]       =tokenizer.tokenize(__lowercase      )
																									__lowerCamelCase   :   str       =tokenizer.convert_tokens_to_ids(__lowercase      )
																									__lowerCamelCase   :   int       =tokenizer.convert_tokens_to_shape_ids(__lowercase      )
																									__lowerCamelCase   :   int       =tokenizer.convert_tokens_to_pronunciation_ids(__lowercase      )
																									__lowerCamelCase   :   Optional[int]       =tokenizer.prepare_for_model(
																									    __lowercase    ,     __lowercase    ,     __lowercase    ,     add_special_tokens=__lowercase      )

																									__lowerCamelCase   :   List[str]       =tokenizer.encode_plus(__lowercase    ,     add_special_tokens=__lowercase      )

																									self.assertEqual(__lowercase    ,     __lowercase      )











"""simple docstring"""





import os
import sys
import tempfile

import torch

from .state import AcceleratorState
from .utils import PrecisionType, PrepareForLaunch, is_mps_available, patch_environment



def 					lowerCAmelCase_      (   SCREAMING_SNAKE_CASE							:				Any     ,    SCREAMING_SNAKE_CASE							:				List[str]=()     ,    SCREAMING_SNAKE_CASE							:				Union[str, Any]=None     ,    SCREAMING_SNAKE_CASE							:				Dict="no"     ,    SCREAMING_SNAKE_CASE							:				int="29500" ):


							'''simple docstring'''







							__lowerCamelCase   :   Optional[int]       =False
							__lowerCamelCase   :   Tuple       =False
							if any(key.startswith('''KAGGLE''' ) for key in os.environ.keys() ):
														__lowerCamelCase   :   Optional[int]       =True
							elif "IPython" in sys.modules:
														__lowerCamelCase   :   int       ='''google.colab''' in str(sys.modules['''IPython'''].get_ipython() )

							try:
														__lowerCamelCase   :   str       =PrecisionType(mixed_precision.lower() )
							except ValueError:
														raise ValueError(
														    F'Unknown mixed_precision mode: {args.mixed_precision.lower()}. Choose between {PrecisionType.list()}.' )

							if (in_colab or in_kaggle) and (os.environ.get('''TPU_NAME'''     ,    SCREAMING_SNAKE_CASE ) is not None):
														# TPU launch
														import torch_xla.distributed.xla_multiprocessing as xmp

														if len(AcceleratorState._shared_state ) > 0:
																					raise ValueError(
																					    '''To train on TPU in Colab or Kaggle Kernel, the `Accelerator` should only be initialized inside '''
																					    '''your training function. Restart your notebook and make sure no cells initializes an '''
																					    '''`Accelerator`.''' )
														if num_processes is None:
																					__lowerCamelCase   :   int       =8

														__lowerCamelCase   :   Optional[Any]       =PrepareForLaunch(SCREAMING_SNAKE_CASE     ,    distributed_type='''TPU''' )
														print(F'Launching a training on {num_processes} TPU cores.' )
														xmp.spawn(SCREAMING_SNAKE_CASE     ,    args=SCREAMING_SNAKE_CASE     ,    nprocs=SCREAMING_SNAKE_CASE     ,    start_method='''fork''' )
							elif in_colab:
														# No need for a distributed launch otherwise as it's either CPU or one GPU.
														if torch.cuda.is_available():
																					print('''Launching training on one GPU.''' )
														else:
																					print('''Launching training on one CPU.''' )
														function(*SCREAMING_SNAKE_CASE )
							else:
														if num_processes is None:
																					raise ValueError(
																					    '''You have to specify the number of GPUs you would like to use, add `num_processes=...` to your call.''' )

														if num_processes > 1:
																					# Multi-GPU launch
																					from torch.multiprocessing import start_processes
																					from torch.multiprocessing.spawn import ProcessRaisedException

																					if len(AcceleratorState._shared_state ) > 0:
																												raise ValueError(
																												    '''To launch a multi-GPU training from your notebook, the `Accelerator` should only be initialized '''
																												    '''inside your training function. Restart your notebook and make sure no cells initializes an '''
																												    '''`Accelerator`.''' )

																					if torch.cuda.is_initialized():
																												raise ValueError(
																												    '''To launch a multi-GPU training from your notebook, you need to avoid running any instruction '''
																												    '''using `torch.cuda` in any cell. Restart your notebook and make sure no cells use any CUDA '''
																												    '''function.''' )

																					# torch.distributed will expect a few environment variable to be here. We set the ones common to each
																					# process here (the other ones will be set be the launcher).
																					with patch_environment(
																					    world_size=SCREAMING_SNAKE_CASE     ,    master_addr='''127.0.01'''     ,    master_port=SCREAMING_SNAKE_CASE     ,    mixed_precision=SCREAMING_SNAKE_CASE ):
																												__lowerCamelCase   :   List[str]       =PrepareForLaunch(SCREAMING_SNAKE_CASE     ,    distributed_type='''MULTI_GPU''' )
																												print(F'Launching training on {num_processes} GPUs.' )
																												try:
																																			start_processes(SCREAMING_SNAKE_CASE     ,    args=SCREAMING_SNAKE_CASE     ,    nprocs=SCREAMING_SNAKE_CASE     ,    start_method='''fork''' )
																												except ProcessRaisedException as e:
																																			if "Cannot re-initialize CUDA in forked subprocess" in e.args[0]:
																																										raise RuntimeError(
																																										    '''CUDA has been initialized before the `notebook_launcher` could create a forked subprocess. '''
																																										    '''This likely stems from an outside import causing issues once the `notebook_launcher()` is called. '''
																																										    '''Please review your imports and test them when running the `notebook_launcher()` to identify '''
																																										    '''which one is problematic.''' ) from e

														else:
																					# No need for a distributed launch otherwise as it's either CPU, GPU or MPS.
																					if is_mps_available():
																												__lowerCamelCase   :   Optional[Any]       ='''1'''
																												print('''Launching training on MPS.''' )
																					elif torch.cuda.is_available():
																												print('''Launching training on one GPU.''' )
																					else:
																												print('''Launching training on CPU.''' )
																					function(*SCREAMING_SNAKE_CASE )



def 					lowerCAmelCase_      (   SCREAMING_SNAKE_CASE							:				str     ,    SCREAMING_SNAKE_CASE							:				Optional[int]=()     ,    SCREAMING_SNAKE_CASE							:				Optional[Any]=2 ):


							'''simple docstring'''







							from torch.multiprocessing import start_processes

							with tempfile.NamedTemporaryFile() as tmp_file:
														# torch.distributed will expect a few environment variable to be here. We set the ones common to each
														# process here (the other ones will be set be the launcher).
														with patch_environment(
														    world_size=SCREAMING_SNAKE_CASE     ,    master_addr='''127.0.01'''     ,    master_port='''29500'''     ,    accelerate_mixed_precision='''no'''     ,    accelerate_debug_rdv_file=tmp_file.name     ,    accelerate_use_cpu='''yes'''     ,    ):
																					__lowerCamelCase   :   Tuple       =PrepareForLaunch(SCREAMING_SNAKE_CASE     ,    debug=SCREAMING_SNAKE_CASE )
																					start_processes(SCREAMING_SNAKE_CASE     ,    args=SCREAMING_SNAKE_CASE     ,    nprocs=SCREAMING_SNAKE_CASE     ,    start_method='''fork''' )




