





"""simple docstring"""


import os
import unittest

from transformers import RealmTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.models.realm.tokenization_realm import RealmTokenizer
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english







@require_tokenizers
class 	__a   (  lowerCAmelCase__	,						unittest.TestCase		):
     SCREAMING_SNAKE_CASE__ :      List[Any]       		=			RealmTokenizer
     SCREAMING_SNAKE_CASE__ :      Dict       		=			RealmTokenizerFast
     SCREAMING_SNAKE_CASE__ :      int       		=			True
     SCREAMING_SNAKE_CASE__ :      Any       		=			True
     SCREAMING_SNAKE_CASE__ :      Optional[int]       		=			filter_non_english



     def 						snake_case_			(       self   ):
            super().setUp()

            _lowerCamelCase    	=					[
                '[UNK]',
                '[CLS]',
                '[SEP]',
                '[PAD]',
                '[MASK]',
                'want',
                '##want',
                '##ed',
                'wa',
                'un',
                'runn',
                '##ing',
                ',',
                'low',
                'lowest',
            ]
            _lowerCamelCase    	=					os.path.join(self.tmpdirname					,     VOCAB_FILES_NAMES['vocab_file']   )
            with open(self.vocab_file					,     'w'					,     encoding='utf-8'   ) as vocab_writer:
                   vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]   )   )



     def 						snake_case_			(       self					,     a__   ):
            _lowerCamelCase    	=					'UNwant\u00E9d,running'
            _lowerCamelCase    	=					'unwanted, running'
            return input_text, output_text



     def 						snake_case_			(       self   ):
            _lowerCamelCase    	=					self.tokenizer_class(self.vocab_file   )

            _lowerCamelCase    	=					tokenizer.tokenize('UNwant\u00E9d,running'   )
            self.assertListEqual(a__					,     ['un', '##want', '##ed', ',', 'runn', '##ing']   )
            self.assertListEqual(tokenizer.convert_tokens_to_ids(a__   )					,     [9, 6, 7, 12, 10, 11]   )



     def 						snake_case_			(       self   ):
            if not self.test_rust_tokenizer:
                   return

            _lowerCamelCase    	=					self.get_tokenizer()
            _lowerCamelCase    	=					self.get_rust_tokenizer()

            _lowerCamelCase    	=					'UNwant\u00E9d,running'

            _lowerCamelCase    	=					tokenizer.tokenize(a__   )
            _lowerCamelCase    	=					rust_tokenizer.tokenize(a__   )
            self.assertListEqual(a__					,     a__   )

            _lowerCamelCase    	=					tokenizer.encode(a__					,     add_special_tokens=a__   )
            _lowerCamelCase    	=					rust_tokenizer.encode(a__					,     add_special_tokens=a__   )
            self.assertListEqual(a__					,     a__   )

            _lowerCamelCase    	=					self.get_rust_tokenizer()
            _lowerCamelCase    	=					tokenizer.encode(a__   )
            _lowerCamelCase    	=					rust_tokenizer.encode(a__   )
            self.assertListEqual(a__					,     a__   )

            # With lower casing
            _lowerCamelCase    	=					self.get_tokenizer(do_lower_case=a__   )
            _lowerCamelCase    	=					self.get_rust_tokenizer(do_lower_case=a__   )

            _lowerCamelCase    	=					'UNwant\u00E9d,running'

            _lowerCamelCase    	=					tokenizer.tokenize(a__   )
            _lowerCamelCase    	=					rust_tokenizer.tokenize(a__   )
            self.assertListEqual(a__					,     a__   )

            _lowerCamelCase    	=					tokenizer.encode(a__					,     add_special_tokens=a__   )
            _lowerCamelCase    	=					rust_tokenizer.encode(a__					,     add_special_tokens=a__   )
            self.assertListEqual(a__					,     a__   )

            _lowerCamelCase    	=					self.get_rust_tokenizer()
            _lowerCamelCase    	=					tokenizer.encode(a__   )
            _lowerCamelCase    	=					rust_tokenizer.encode(a__   )
            self.assertListEqual(a__					,     a__   )



     def 						snake_case_			(       self   ):
            _lowerCamelCase    	=					BasicTokenizer()

            self.assertListEqual(tokenizer.tokenize('ah\u535A\u63A8zz'   )					,     ['ah', '\u535A', '\u63A8', 'zz']   )



     def 						snake_case_			(       self   ):
            _lowerCamelCase    	=					BasicTokenizer(do_lower_case=a__   )

            self.assertListEqual(
                tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '   )					,     ['hello', '!', 'how', 'are', 'you', '?']   )
            self.assertListEqual(tokenizer.tokenize('H\u00E9llo'   )					,     ['hello']   )



     def 						snake_case_			(       self   ):
            _lowerCamelCase    	=					BasicTokenizer(do_lower_case=a__					,     strip_accents=a__   )

            self.assertListEqual(
                tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '   )					,     ['hällo', '!', 'how', 'are', 'you', '?']   )
            self.assertListEqual(tokenizer.tokenize('H\u00E9llo'   )					,     ['h\u00E9llo']   )



     def 						snake_case_			(       self   ):
            _lowerCamelCase    	=					BasicTokenizer(do_lower_case=a__					,     strip_accents=a__   )

            self.assertListEqual(
                tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '   )					,     ['hallo', '!', 'how', 'are', 'you', '?']   )
            self.assertListEqual(tokenizer.tokenize('H\u00E9llo'   )					,     ['hello']   )



     def 						snake_case_			(       self   ):
            _lowerCamelCase    	=					BasicTokenizer(do_lower_case=a__   )

            self.assertListEqual(
                tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '   )					,     ['hallo', '!', 'how', 'are', 'you', '?']   )
            self.assertListEqual(tokenizer.tokenize('H\u00E9llo'   )					,     ['hello']   )



     def 						snake_case_			(       self   ):
            _lowerCamelCase    	=					BasicTokenizer(do_lower_case=a__   )

            self.assertListEqual(
                tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '   )					,     ['HeLLo', '!', 'how', 'Are', 'yoU', '?']   )



     def 						snake_case_			(       self   ):
            _lowerCamelCase    	=					BasicTokenizer(do_lower_case=a__					,     strip_accents=a__   )

            self.assertListEqual(
                tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '   )					,     ['HäLLo', '!', 'how', 'Are', 'yoU', '?']   )



     def 						snake_case_			(       self   ):
            _lowerCamelCase    	=					BasicTokenizer(do_lower_case=a__					,     strip_accents=a__   )

            self.assertListEqual(
                tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '   )					,     ['HaLLo', '!', 'how', 'Are', 'yoU', '?']   )



     def 						snake_case_			(       self   ):
            _lowerCamelCase    	=					BasicTokenizer(do_lower_case=a__					,     never_split=['[UNK]']   )

            self.assertListEqual(
                tokenizer.tokenize(' \tHeLLo!how  \n Are yoU? [UNK]'   )					,     ['HeLLo', '!', 'how', 'Are', 'yoU', '?', '[UNK]']   )



     def 						snake_case_			(       self   ):
            _lowerCamelCase    	=					['[UNK]', '[CLS]', '[SEP]', 'want', '##want', '##ed', 'wa', 'un', 'runn', '##ing']

            _lowerCamelCase    	=					{}
            for i, token in enumerate(a__   ):
                   _lowerCamelCase    	=					i
            _lowerCamelCase    	=					WordpieceTokenizer(vocab=a__					,     unk_token='[UNK]'   )

            self.assertListEqual(tokenizer.tokenize(''   )					,     []   )

            self.assertListEqual(tokenizer.tokenize('unwanted running'   )					,     ['un', '##want', '##ed', 'runn', '##ing']   )

            self.assertListEqual(tokenizer.tokenize('unwantedX running'   )					,     ['[UNK]', 'runn', '##ing']   )



     def 						snake_case_			(       self   ):
            self.assertTrue(_is_whitespace(' '   )   )
            self.assertTrue(_is_whitespace('\t'   )   )
            self.assertTrue(_is_whitespace('\r'   )   )
            self.assertTrue(_is_whitespace('\n'   )   )
            self.assertTrue(_is_whitespace('\u00A0'   )   )

            self.assertFalse(_is_whitespace('A'   )   )
            self.assertFalse(_is_whitespace('-'   )   )



     def 						snake_case_			(       self   ):
            self.assertTrue(_is_control('\u0005'   )   )

            self.assertFalse(_is_control('A'   )   )
            self.assertFalse(_is_control(' '   )   )
            self.assertFalse(_is_control('\t'   )   )
            self.assertFalse(_is_control('\r'   )   )



     def 						snake_case_			(       self   ):
            self.assertTrue(_is_punctuation('-'   )   )
            self.assertTrue(_is_punctuation('$'   )   )
            self.assertTrue(_is_punctuation('`'   )   )
            self.assertTrue(_is_punctuation('.'   )   )

            self.assertFalse(_is_punctuation('A'   )   )
            self.assertFalse(_is_punctuation(' '   )   )



     def 						snake_case_			(       self   ):
            _lowerCamelCase    	=					self.get_tokenizer()

            # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
            self.assertListEqual([tokenizer.tokenize(a__   ) for t in ['Test', '\xad', 'test']]					,     [['[UNK]'], [], ['[UNK]']]   )

            if self.test_rust_tokenizer:
                   _lowerCamelCase    	=					self.get_rust_tokenizer()
                   self.assertListEqual(
                       [rust_tokenizer.tokenize(a__   ) for t in ['Test', '\xad', 'test']]					,     [['[UNK]'], [], ['[UNK]']]   )



     @slow
     def 						snake_case_			(       self   ):
            _lowerCamelCase    	=					self.tokenizer_class.from_pretrained('bert-base-uncased'   )

            _lowerCamelCase    	=					tokenizer.encode('sequence builders'					,     add_special_tokens=a__   )
            _lowerCamelCase    	=					tokenizer.encode('multi-sequence build'					,     add_special_tokens=a__   )

            _lowerCamelCase    	=					tokenizer.build_inputs_with_special_tokens(a__   )
            _lowerCamelCase    	=					tokenizer.build_inputs_with_special_tokens(a__					,     a__   )

            assert encoded_sentence == [1_01] + text + [1_02]
            assert encoded_pair == [1_01] + text + [1_02] + text_a + [1_02]



     def 						snake_case_			(       self   ):
            for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                   with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})'   ):
                          _lowerCamelCase    	=					self.rust_tokenizer_class.from_pretrained(a__					,     **a__   )

                          _lowerCamelCase    	=					F'A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'
                          _lowerCamelCase    	=					tokenizer_r.encode_plus(
                              a__					,     return_attention_mask=a__					,     return_token_type_ids=a__					,     return_offsets_mapping=a__					,     add_special_tokens=a__					,     )

                          _lowerCamelCase    	=					tokenizer_r.do_lower_case if hasattr(a__					,     'do_lower_case'   ) else False
                          _lowerCamelCase    	=					(
                              [
                                  ((0, 0), tokenizer_r.cls_token),
                                  ((0, 1), 'A'),
                                  ((1, 2), ','),
                                  ((3, 5), 'na'),
                                  ((5, 6), '##ï'),
                                  ((6, 8), '##ve'),
                                  ((9, 15), tokenizer_r.mask_token),
                                  ((16, 21), 'Allen'),
                                  ((21, 23), '##NL'),
                                  ((23, 24), '##P'),
                                  ((25, 33), 'sentence'),
                                  ((33, 34), '.'),
                                  ((0, 0), tokenizer_r.sep_token),
                              ]
                              if not do_lower_case
                              else [
                                  ((0, 0), tokenizer_r.cls_token),
                                  ((0, 1), 'a'),
                                  ((1, 2), ','),
                                  ((3, 8), 'naive'),
                                  ((9, 15), tokenizer_r.mask_token),
                                  ((16, 21), 'allen'),
                                  ((21, 23), '##nl'),
                                  ((23, 24), '##p'),
                                  ((25, 33), 'sentence'),
                                  ((33, 34), '.'),
                                  ((0, 0), tokenizer_r.sep_token),
                              ]
                          )

                          self.assertEqual(
                              [e[1] for e in expected_results]					,     tokenizer_r.convert_ids_to_tokens(tokens['input_ids']   )   )
                          self.assertEqual([e[0] for e in expected_results]					,     tokens['offset_mapping']   )



     @slow
     def 						snake_case_			(       self   ):
            for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                   with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})'   ):
                          _lowerCamelCase    	=					self.rust_tokenizer_class.from_pretrained(a__					,     **a__   )
                          _lowerCamelCase    	=					self.tokenizer_class.from_pretrained(a__					,     **a__   )
                          _lowerCamelCase    	=					[['Hello world!', 'Nice to meet you!'], ['The cute cat.', 'The adorable dog.']]

                          _lowerCamelCase    	=					tokenizer_r.batch_encode_candidates(a__					,     max_length=10					,     return_tensors='np'   )
                          _lowerCamelCase    	=					tokenizer_p.batch_encode_candidates(a__					,     max_length=10					,     return_tensors='np'   )

                          _lowerCamelCase    	=					(2, 2, 10)

                          self.assertEqual(encoded_sentence_r['input_ids'].shape					,     a__   )
                          self.assertEqual(encoded_sentence_r['attention_mask'].shape					,     a__   )
                          self.assertEqual(encoded_sentence_r['token_type_ids'].shape					,     a__   )

                          self.assertEqual(encoded_sentence_p['input_ids'].shape					,     a__   )
                          self.assertEqual(encoded_sentence_p['attention_mask'].shape					,     a__   )
                          self.assertEqual(encoded_sentence_p['token_type_ids'].shape					,     a__   )













"""simple docstring"""


import random
from typing import List, Optional, Union

import numpy as np
import torch

from ..state import AcceleratorState
from .constants import CUDA_DISTRIBUTED_TYPES
from .dataclasses import DistributedType, RNGType
from .imports import is_npu_available, is_tpu_available, is_xpu_available


if is_tpu_available(check_device=False):
      import torch_xla.core.xla_model as xm





def SCREAMING_SNAKE_CASE_   (       snake_case   :					int						,  snake_case   :					bool = False		)->		Any:
       if device_specific:
              seed += AcceleratorState().process_index
       random.seed(snake_case		)
       np.random.seed(snake_case		)
       torch.manual_seed(snake_case		)
       if is_xpu_available():
              torch.xpu.manual_seed_all(snake_case		)
       elif is_npu_available():
              torch.npu.manual_seed_all(snake_case		)
       else:
              torch.cuda.manual_seed_all(snake_case		)
       # ^^ safe to call this function even if cuda is not available
       if is_tpu_available():
              xm.set_rng_state(snake_case		)





def SCREAMING_SNAKE_CASE_   (       snake_case   :					Optional[RNGType] = None						,  snake_case   :					Optional[torch.Generator] = None		)->		str:
       # Get the proper rng state
       if rng_type == RNGType.TORCH:
              _lowerCamelCase    	=					torch.get_rng_state()
       elif rng_type == RNGType.CUDA:
              _lowerCamelCase    	=					torch.cuda.get_rng_state()
       elif rng_type == RNGType.XLA:
              assert is_tpu_available(), "Can't synchronize XLA seeds on an environment without TPUs."
              _lowerCamelCase    	=					torch.tensor(xm.get_rng_state()		)
       elif rng_type == RNGType.NPU:
              assert is_npu_available(), "Can't synchronize NPU seeds on an environment without NPUs."
              _lowerCamelCase    	=					torch.npu.get_rng_state()
       elif rng_type == RNGType.XPU:
              assert is_xpu_available(), "Can't synchronize XPU seeds on an environment without XPUs."
              _lowerCamelCase    	=					torch.xpu.get_rng_state()
       elif rng_type == RNGType.GENERATOR:
              assert generator is not None, "Need a generator to synchronize its seed."
              _lowerCamelCase    	=					generator.get_state()

       # Broadcast the rng state from device 0 to other devices
       _lowerCamelCase    	=					AcceleratorState()
       if state.distributed_type == DistributedType.TPU:
              _lowerCamelCase    	=					rng_state.to(xm.xla_device()		)
              xm.collective_broadcast([rng_state]		)
              xm.mark_step()
              _lowerCamelCase    	=					rng_state.cpu()
       elif (
           state.distributed_type in CUDA_DISTRIBUTED_TYPES
           or state.distributed_type == DistributedType.MULTI_NPU
           or state.distributed_type == DistributedType.MULTI_XPU
       ):
              _lowerCamelCase    	=					rng_state.to(state.device		)
              torch.distributed.broadcast(snake_case						,  0		)
              _lowerCamelCase    	=					rng_state.cpu()
       elif state.distributed_type == DistributedType.MULTI_CPU:
              torch.distributed.broadcast(snake_case						,  0		)

       # Set the broadcast rng state
       if rng_type == RNGType.TORCH:
              torch.set_rng_state(snake_case		)
       elif rng_type == RNGType.CUDA:
              torch.cuda.set_rng_state(snake_case		)
       elif rng_type == RNGType.NPU:
              torch.npu.set_rng_state(snake_case		)
       elif rng_type == RNGType.XPU:
              torch.xpu.set_rng_state(snake_case		)
       elif rng_type == RNGType.XLA:
              xm.set_rng_state(rng_state.item()		)
       elif rng_type == RNGType.GENERATOR:
              generator.set_state(snake_case		)





def SCREAMING_SNAKE_CASE_   (       snake_case   :					List[Union[str, RNGType]]						,  snake_case   :					Optional[torch.Generator] = None		)->		int:
       for rng_type in rng_types:
              synchronize_rng_state(RNGType(snake_case		)						,  generator=snake_case		)






