


'''simple docstring'''



import os
import unittest

from transformers import ElectraTokenizerFast
from transformers.models.electra.tokenization_electra import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    ElectraTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english






@require_tokenizers
class        UpperCamelCase__(   lowerCAmelCase__						,				unittest.TestCase       ):
				__magic_name__				:      Tuple  		=       ElectraTokenizer
				__magic_name__				:      Dict  		=       ElectraTokenizerFast
				__magic_name__				:      Optional[Any]  		=       True
				__magic_name__				:      Optional[Any]  		=       True
				__magic_name__				:      Any  		=       filter_non_english




				def      a__(		self			: Optional[int]      )->     Optional[int]:

										"""simple docstring"""




										super().setUp()

										UpperCAmelCase     				=	[
										    '''[UNK]''',
										    '''[CLS]''',
										    '''[SEP]''',
										    '''[PAD]''',
										    '''[MASK]''',
										    '''want''',
										    '''##want''',
										    '''##ed''',
										    '''wa''',
										    '''un''',
										    '''runn''',
										    '''##ing''',
										    ''',''',
										    '''low''',
										    '''lowest''',
										]
										UpperCAmelCase     				=	os.path.join(self.tmpdirname      ,    VOCAB_FILES_NAMES['''vocab_file''']      )
										with open(self.vocab_file      ,    '''w'''      ,    encoding='''utf-8'''      ) as vocab_writer:
																vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]      )      )




				def      a__(		self			: Union[str, Any]      ,    lowerCAmelCase			: Tuple      )->     Optional[Any]:

										"""simple docstring"""




										UpperCAmelCase     				=	'''UNwant\u00E9d,running'''
										UpperCAmelCase     				=	'''unwanted, running'''
										return input_text, output_text




				def      a__(		self			: Any      )->     Optional[Any]:

										"""simple docstring"""




										UpperCAmelCase     				=	self.tokenizer_class(self.vocab_file      )

										UpperCAmelCase     				=	tokenizer.tokenize('''UNwant\u00E9d,running'''      )
										self.assertListEqual(a__      ,    ['''un''', '''##want''', '''##ed''', ''',''', '''runn''', '''##ing''']      )
										self.assertListEqual(tokenizer.convert_tokens_to_ids(a__      )      ,    [9, 6, 7, 12, 10, 11]      )




				def      a__(		self			: Tuple      )->     str:

										"""simple docstring"""




										if not self.test_rust_tokenizer:
																return

										UpperCAmelCase     				=	self.get_tokenizer()
										UpperCAmelCase     				=	self.get_rust_tokenizer()

										UpperCAmelCase     				=	'''UNwant\u00E9d,running'''

										UpperCAmelCase     				=	tokenizer.tokenize(a__      )
										UpperCAmelCase     				=	rust_tokenizer.tokenize(a__      )
										self.assertListEqual(a__      ,    a__      )

										UpperCAmelCase     				=	tokenizer.encode(a__      ,    add_special_tokens=a__      )
										UpperCAmelCase     				=	rust_tokenizer.encode(a__      ,    add_special_tokens=a__      )
										self.assertListEqual(a__      ,    a__      )

										UpperCAmelCase     				=	self.get_rust_tokenizer()
										UpperCAmelCase     				=	tokenizer.encode(a__      )
										UpperCAmelCase     				=	rust_tokenizer.encode(a__      )
										self.assertListEqual(a__      ,    a__      )

										# With lower casing
										UpperCAmelCase     				=	self.get_tokenizer(do_lower_case=a__      )
										UpperCAmelCase     				=	self.get_rust_tokenizer(do_lower_case=a__      )

										UpperCAmelCase     				=	'''UNwant\u00E9d,running'''

										UpperCAmelCase     				=	tokenizer.tokenize(a__      )
										UpperCAmelCase     				=	rust_tokenizer.tokenize(a__      )
										self.assertListEqual(a__      ,    a__      )

										UpperCAmelCase     				=	tokenizer.encode(a__      ,    add_special_tokens=a__      )
										UpperCAmelCase     				=	rust_tokenizer.encode(a__      ,    add_special_tokens=a__      )
										self.assertListEqual(a__      ,    a__      )

										UpperCAmelCase     				=	self.get_rust_tokenizer()
										UpperCAmelCase     				=	tokenizer.encode(a__      )
										UpperCAmelCase     				=	rust_tokenizer.encode(a__      )
										self.assertListEqual(a__      ,    a__      )




				def      a__(		self			: Optional[Any]      )->     Optional[int]:

										"""simple docstring"""




										UpperCAmelCase     				=	BasicTokenizer()

										self.assertListEqual(tokenizer.tokenize('''ah\u535A\u63A8zz'''      )      ,    ['''ah''', '''\u535A''', '''\u63A8''', '''zz''']      )




				def      a__(		self			: Union[str, Any]      )->     Union[str, Any]:

										"""simple docstring"""




										UpperCAmelCase     				=	BasicTokenizer(do_lower_case=a__      )

										self.assertListEqual(
										    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''      )      ,    ['''hello''', '''!''', '''how''', '''are''', '''you''', '''?''']      )
										self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''      )      ,    ['''hello''']      )




				def      a__(		self			: List[str]      )->     Dict:

										"""simple docstring"""




										UpperCAmelCase     				=	BasicTokenizer(do_lower_case=a__      ,    strip_accents=a__      )

										self.assertListEqual(
										    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''      )      ,    ['''hällo''', '''!''', '''how''', '''are''', '''you''', '''?''']      )
										self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''      )      ,    ['''h\u00E9llo''']      )




				def      a__(		self			: Dict      )->     List[str]:

										"""simple docstring"""




										UpperCAmelCase     				=	BasicTokenizer(do_lower_case=a__      ,    strip_accents=a__      )

										self.assertListEqual(
										    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''      )      ,    ['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']      )
										self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''      )      ,    ['''hello''']      )




				def      a__(		self			: int      )->     Any:

										"""simple docstring"""




										UpperCAmelCase     				=	BasicTokenizer(do_lower_case=a__      )

										self.assertListEqual(
										    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''      )      ,    ['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']      )
										self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''      )      ,    ['''hello''']      )




				def      a__(		self			: Optional[Any]      )->     Optional[int]:

										"""simple docstring"""




										UpperCAmelCase     				=	BasicTokenizer(do_lower_case=a__      )

										self.assertListEqual(
										    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''      )      ,    ['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']      )




				def      a__(		self			: int      )->     List[Any]:

										"""simple docstring"""




										UpperCAmelCase     				=	BasicTokenizer(do_lower_case=a__      ,    strip_accents=a__      )

										self.assertListEqual(
										    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''      )      ,    ['''HäLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']      )




				def      a__(		self			: Union[str, Any]      )->     Tuple:

										"""simple docstring"""




										UpperCAmelCase     				=	BasicTokenizer(do_lower_case=a__      ,    strip_accents=a__      )

										self.assertListEqual(
										    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''      )      ,    ['''HaLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']      )




				def      a__(		self			: Optional[Any]      )->     Any:

										"""simple docstring"""




										UpperCAmelCase     				=	BasicTokenizer(do_lower_case=a__      ,    never_split=['''[UNK]''']      )

										self.assertListEqual(
										    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU? [UNK]'''      )      ,    ['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''', '''[UNK]''']      )




				def      a__(		self			: Tuple      )->     Union[str, Any]:

										"""simple docstring"""




										UpperCAmelCase     				=	['''[UNK]''', '''[CLS]''', '''[SEP]''', '''want''', '''##want''', '''##ed''', '''wa''', '''un''', '''runn''', '''##ing''']

										UpperCAmelCase     				=	{}
										for i, token in enumerate(a__      ):
																UpperCAmelCase     				=	i
										UpperCAmelCase     				=	WordpieceTokenizer(vocab=a__      ,    unk_token='''[UNK]'''      )

										self.assertListEqual(tokenizer.tokenize(''''''      )      ,    []      )

										self.assertListEqual(tokenizer.tokenize('''unwanted running'''      )      ,    ['''un''', '''##want''', '''##ed''', '''runn''', '''##ing''']      )

										self.assertListEqual(tokenizer.tokenize('''unwantedX running'''      )      ,    ['''[UNK]''', '''runn''', '''##ing''']      )




				def      a__(		self			: str      )->     List[Any]:

										"""simple docstring"""




										self.assertTrue(_is_whitespace(''' '''      )      )
										self.assertTrue(_is_whitespace('''\t'''      )      )
										self.assertTrue(_is_whitespace('''\r'''      )      )
										self.assertTrue(_is_whitespace('''\n'''      )      )
										self.assertTrue(_is_whitespace('''\u00A0'''      )      )

										self.assertFalse(_is_whitespace('''A'''      )      )
										self.assertFalse(_is_whitespace('''-'''      )      )




				def      a__(		self			: Tuple      )->     str:

										"""simple docstring"""




										self.assertTrue(_is_control('''\u0005'''      )      )

										self.assertFalse(_is_control('''A'''      )      )
										self.assertFalse(_is_control(''' '''      )      )
										self.assertFalse(_is_control('''\t'''      )      )
										self.assertFalse(_is_control('''\r'''      )      )




				def      a__(		self			: List[Any]      )->     List[Any]:

										"""simple docstring"""




										self.assertTrue(_is_punctuation('''-'''      )      )
										self.assertTrue(_is_punctuation('''$'''      )      )
										self.assertTrue(_is_punctuation('''`'''      )      )
										self.assertTrue(_is_punctuation('''.'''      )      )

										self.assertFalse(_is_punctuation('''A'''      )      )
										self.assertFalse(_is_punctuation(''' '''      )      )




				def      a__(		self			: Optional[Any]      )->     List[str]:

										"""simple docstring"""




										UpperCAmelCase     				=	self.get_tokenizer()
										UpperCAmelCase     				=	self.get_rust_tokenizer()

										# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
										self.assertListEqual([tokenizer.tokenize(a__      ) for t in ['''Test''', '''\xad''', '''test''']]      ,    [['''[UNK]'''], [], ['''[UNK]''']]      )

										self.assertListEqual(
										    [rust_tokenizer.tokenize(a__      ) for t in ['''Test''', '''\xad''', '''test''']]      ,    [['''[UNK]'''], [], ['''[UNK]''']]      )




				@slow
				def      a__(		self			: Union[str, Any]      )->     Tuple:

										"""simple docstring"""




										UpperCAmelCase     				=	self.tokenizer_class.from_pretrained('''google/electra-base-discriminator'''      )

										UpperCAmelCase     				=	tokenizer.encode('''sequence builders'''      ,    add_special_tokens=a__      )
										UpperCAmelCase     				=	tokenizer.encode('''multi-sequence build'''      ,    add_special_tokens=a__      )

										UpperCAmelCase     				=	tokenizer.build_inputs_with_special_tokens(a__      )
										UpperCAmelCase     				=	tokenizer.build_inputs_with_special_tokens(a__      ,    a__      )

										assert encoded_sentence == [101] + text + [102]
										assert encoded_pair == [101] + text + [102] + text_a + [102]




				def      a__(		self			: int      )->     Any:

										"""simple docstring"""




										for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""      ):
																						UpperCAmelCase     				=	self.rust_tokenizer_class.from_pretrained(a__      ,    **a__      )

																						UpperCAmelCase     				=	F"""A, naïve {tokenizer_r.mask_token} AllenNLP sentence."""
																						UpperCAmelCase     				=	tokenizer_r.encode_plus(
																						    a__      ,    return_attention_mask=a__      ,    return_token_type_ids=a__      ,    return_offsets_mapping=a__      ,    add_special_tokens=a__      ,    )

																						UpperCAmelCase     				=	tokenizer_r.do_lower_case if hasattr(a__      ,    '''do_lower_case'''      ) else False
																						UpperCAmelCase     				=	(
																						    [
																						        ((0, 0), tokenizer_r.cls_token),
																						        ((0, 1), '''A'''),
																						        ((1, 2), ''','''),
																						        ((3, 5), '''na'''),
																						        ((5, 6), '''##ï'''),
																						        ((6, 8), '''##ve'''),
																						        ((9, 15), tokenizer_r.mask_token),
																						        ((16, 21), '''Allen'''),
																						        ((21, 23), '''##NL'''),
																						        ((23, 24), '''##P'''),
																						        ((25, 33), '''sentence'''),
																						        ((33, 34), '''.'''),
																						        ((0, 0), tokenizer_r.sep_token),
																						    ]
																						    if not do_lower_case
																						    else [
																						        ((0, 0), tokenizer_r.cls_token),
																						        ((0, 1), '''a'''),
																						        ((1, 2), ''','''),
																						        ((3, 8), '''naive'''),
																						        ((9, 15), tokenizer_r.mask_token),
																						        ((16, 21), '''allen'''),
																						        ((21, 23), '''##nl'''),
																						        ((23, 24), '''##p'''),
																						        ((25, 33), '''sentence'''),
																						        ((33, 34), '''.'''),
																						        ((0, 0), tokenizer_r.sep_token),
																						    ]
																						)

																						self.assertEqual(
																						    [e[1] for e in expected_results]      ,    tokenizer_r.convert_ids_to_tokens(tokens['''input_ids''']      )      )
																						self.assertEqual([e[0] for e in expected_results]      ,    tokens['''offset_mapping''']      )





				def      a__(		self			: Tuple      )->     str:

										"""simple docstring"""




										UpperCAmelCase     				=	['''的''', '''人''', '''有''']
										UpperCAmelCase     				=	''''''.join(a__      )
										for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""      ):
																						UpperCAmelCase     				=	True
																						UpperCAmelCase     				=	self.tokenizer_class.from_pretrained(a__      ,    **a__      )
																						UpperCAmelCase     				=	self.rust_tokenizer_class.from_pretrained(a__      ,    **a__      )

																						UpperCAmelCase     				=	tokenizer_p.encode(a__      ,    add_special_tokens=a__      )
																						UpperCAmelCase     				=	tokenizer_r.encode(a__      ,    add_special_tokens=a__      )

																						UpperCAmelCase     				=	tokenizer_r.convert_ids_to_tokens(a__      )
																						UpperCAmelCase     				=	tokenizer_p.convert_ids_to_tokens(a__      )

																						# it is expected that each Chinese character is not preceded by "##"
																						self.assertListEqual(a__      ,    a__      )
																						self.assertListEqual(a__      ,    a__      )

																						UpperCAmelCase     				=	False
																						UpperCAmelCase     				=	self.rust_tokenizer_class.from_pretrained(a__      ,    **a__      )
																						UpperCAmelCase     				=	self.tokenizer_class.from_pretrained(a__      ,    **a__      )

																						UpperCAmelCase     				=	tokenizer_r.encode(a__      ,    add_special_tokens=a__      )
																						UpperCAmelCase     				=	tokenizer_p.encode(a__      ,    add_special_tokens=a__      )

																						UpperCAmelCase     				=	tokenizer_r.convert_ids_to_tokens(a__      )
																						UpperCAmelCase     				=	tokenizer_p.convert_ids_to_tokens(a__      )

																						# it is expected that only the first Chinese character is not preceded by "##".
																						UpperCAmelCase     				=	[
																						    F"""##{token}""" if idx != 0 else token for idx, token in enumerate(a__      )
																						]
																						self.assertListEqual(a__      ,    a__      )
																						self.assertListEqual(a__      ,    a__      )












"""simple docstring"""


import random
from typing import List, Optional, Union

import numpy as np
import torch

from ..state import AcceleratorState
from .constants import CUDA_DISTRIBUTED_TYPES
from .dataclasses import DistributedType, RNGType
from .imports import is_npu_available, is_tpu_available, is_xpu_available


if is_tpu_available(check_device=False):
      import torch_xla.core.xla_model as xm





def SCREAMING_SNAKE_CASE_   (       snake_case   :					int						,  snake_case   :					bool = False		)->		Any:
       if device_specific:
              seed += AcceleratorState().process_index
       random.seed(snake_case		)
       np.random.seed(snake_case		)
       torch.manual_seed(snake_case		)
       if is_xpu_available():
              torch.xpu.manual_seed_all(snake_case		)
       elif is_npu_available():
              torch.npu.manual_seed_all(snake_case		)
       else:
              torch.cuda.manual_seed_all(snake_case		)
       # ^^ safe to call this function even if cuda is not available
       if is_tpu_available():
              xm.set_rng_state(snake_case		)





def SCREAMING_SNAKE_CASE_   (       snake_case   :					Optional[RNGType] = None						,  snake_case   :					Optional[torch.Generator] = None		)->		str:
       # Get the proper rng state
       if rng_type == RNGType.TORCH:
              _lowerCamelCase    	=					torch.get_rng_state()
       elif rng_type == RNGType.CUDA:
              _lowerCamelCase    	=					torch.cuda.get_rng_state()
       elif rng_type == RNGType.XLA:
              assert is_tpu_available(), "Can't synchronize XLA seeds on an environment without TPUs."
              _lowerCamelCase    	=					torch.tensor(xm.get_rng_state()		)
       elif rng_type == RNGType.NPU:
              assert is_npu_available(), "Can't synchronize NPU seeds on an environment without NPUs."
              _lowerCamelCase    	=					torch.npu.get_rng_state()
       elif rng_type == RNGType.XPU:
              assert is_xpu_available(), "Can't synchronize XPU seeds on an environment without XPUs."
              _lowerCamelCase    	=					torch.xpu.get_rng_state()
       elif rng_type == RNGType.GENERATOR:
              assert generator is not None, "Need a generator to synchronize its seed."
              _lowerCamelCase    	=					generator.get_state()

       # Broadcast the rng state from device 0 to other devices
       _lowerCamelCase    	=					AcceleratorState()
       if state.distributed_type == DistributedType.TPU:
              _lowerCamelCase    	=					rng_state.to(xm.xla_device()		)
              xm.collective_broadcast([rng_state]		)
              xm.mark_step()
              _lowerCamelCase    	=					rng_state.cpu()
       elif (
           state.distributed_type in CUDA_DISTRIBUTED_TYPES
           or state.distributed_type == DistributedType.MULTI_NPU
           or state.distributed_type == DistributedType.MULTI_XPU
       ):
              _lowerCamelCase    	=					rng_state.to(state.device		)
              torch.distributed.broadcast(snake_case						,  0		)
              _lowerCamelCase    	=					rng_state.cpu()
       elif state.distributed_type == DistributedType.MULTI_CPU:
              torch.distributed.broadcast(snake_case						,  0		)

       # Set the broadcast rng state
       if rng_type == RNGType.TORCH:
              torch.set_rng_state(snake_case		)
       elif rng_type == RNGType.CUDA:
              torch.cuda.set_rng_state(snake_case		)
       elif rng_type == RNGType.NPU:
              torch.npu.set_rng_state(snake_case		)
       elif rng_type == RNGType.XPU:
              torch.xpu.set_rng_state(snake_case		)
       elif rng_type == RNGType.XLA:
              xm.set_rng_state(rng_state.item()		)
       elif rng_type == RNGType.GENERATOR:
              generator.set_state(snake_case		)





def SCREAMING_SNAKE_CASE_   (       snake_case   :					List[Union[str, RNGType]]						,  snake_case   :					Optional[torch.Generator] = None		)->		int:
       for rng_type in rng_types:
              synchronize_rng_state(RNGType(snake_case		)						,  generator=snake_case		)






