






"""simple docstring"""



import random
from typing import List, Optional, Union

import numpy as np
import torch

from ..state import AcceleratorState
from .constants import CUDA_DISTRIBUTED_TYPES
from .dataclasses import DistributedType, RNGType
from .imports import is_npu_available, is_tpu_available, is_xpu_available


if is_tpu_available(check_device=False):
 import torch_xla.core.xla_model as xm


def      SCREAMING_SNAKE_CASE__    (   snake_case    :      Union[str, Any]       , snake_case    :      List[Any] = False							)->						Optional[Any]:






     '''simple docstring'''



     if device_specific:
          seed += AcceleratorState().process_index
     random.seed(lowerCAmelCase_							)
     np.random.seed(lowerCAmelCase_							)
     torch.manual_seed(lowerCAmelCase_							)
     if is_xpu_available():
          torch.xpu.manual_seed_all(lowerCAmelCase_							)
     elif is_npu_available():
          torch.npu.manual_seed_all(lowerCAmelCase_							)
     else:
          torch.cuda.manual_seed_all(lowerCAmelCase_							)
     # ^^ safe to call this function even if cuda is not available
     if is_tpu_available():
          xm.set_rng_state(lowerCAmelCase_							)


def      SCREAMING_SNAKE_CASE__    (   snake_case    :      List[str] = None       , snake_case    :      Dict = None							)->						Optional[Any]:






     '''simple docstring'''



     if rng_type == RNGType.TORCH:
          UpperCAmelCase__		:  Dict				=				torch.get_rng_state()
     elif rng_type == RNGType.CUDA:
          UpperCAmelCase__		:  Union[str, Any]				=				torch.cuda.get_rng_state()
     elif rng_type == RNGType.XLA:
          assert is_tpu_available(), "Can't synchronize XLA seeds on an environment without TPUs."
          UpperCAmelCase__		:  Optional[int]				=				torch.tensor(xm.get_rng_state()							)
     elif rng_type == RNGType.NPU:
          assert is_npu_available(), "Can't synchronize NPU seeds on an environment without NPUs."
          UpperCAmelCase__		:  Union[str, Any]				=				torch.npu.get_rng_state()
     elif rng_type == RNGType.XPU:
          assert is_xpu_available(), "Can't synchronize XPU seeds on an environment without XPUs."
          UpperCAmelCase__		:  Optional[int]				=				torch.xpu.get_rng_state()
     elif rng_type == RNGType.GENERATOR:
          assert generator is not None, "Need a generator to synchronize its seed."
          UpperCAmelCase__		:  Dict				=				generator.get_state()

     # Broadcast the rng state from device 0 to other devices
     UpperCAmelCase__		:  List[Any]				=				AcceleratorState()
     if state.distributed_type == DistributedType.TPU:
          UpperCAmelCase__		:  Tuple				=				rng_state.to(xm.xla_device()							)
          xm.collective_broadcast([rng_state]							)
          xm.mark_step()
          UpperCAmelCase__		:  Optional[int]				=				rng_state.cpu()
     elif (
         state.distributed_type in CUDA_DISTRIBUTED_TYPES
         or state.distributed_type == DistributedType.MULTI_NPU
         or state.distributed_type == DistributedType.MULTI_XPU
     ):
          UpperCAmelCase__		:  Tuple				=				rng_state.to(state.device							)
          torch.distributed.broadcast(lowerCAmelCase_       , 0							)
          UpperCAmelCase__		:  List[str]				=				rng_state.cpu()
     elif state.distributed_type == DistributedType.MULTI_CPU:
          torch.distributed.broadcast(lowerCAmelCase_       , 0							)

     # Set the broadcast rng state
     if rng_type == RNGType.TORCH:
          torch.set_rng_state(lowerCAmelCase_							)
     elif rng_type == RNGType.CUDA:
          torch.cuda.set_rng_state(lowerCAmelCase_							)
     elif rng_type == RNGType.NPU:
          torch.npu.set_rng_state(lowerCAmelCase_							)
     elif rng_type == RNGType.XPU:
          torch.xpu.set_rng_state(lowerCAmelCase_							)
     elif rng_type == RNGType.XLA:
          xm.set_rng_state(rng_state.item()							)
     elif rng_type == RNGType.GENERATOR:
          generator.set_state(lowerCAmelCase_							)



def      SCREAMING_SNAKE_CASE__    (   snake_case    :      Union[str, Any]       , snake_case    :      Tuple = None							)->						Any:






     '''simple docstring'''



     for rng_type in rng_types:
          synchronize_rng_state(RNGType(lowerCAmelCase_							)       , generator=lowerCAmelCase_							)




'''simple docstring'''

import concurrent.futures
import json
import os
import shutil
import tempfile
import unittest

from transformers import AutoTokenizer, PreTrainedTokenizerFast
from transformers.testing_utils import require_tokenizers

from ..test_tokenization_common import TokenizerTesterMixin







@require_tokenizers
class 	__lowerCAmelCase    (  __a       ,					unittest.TestCase				):
     snake_case   :    Dict							      =      PreTrainedTokenizerFast
     snake_case   :    Optional[Any]							      =      False
     snake_case   :    List[str]							      =      True
     snake_case   :    Any							      =      """tokenizer_file"""



     def     snake_case_ (self	):
          _UpperCAmelCase					:  Dict         =    False  # because we don't have pretrained_vocab_files_map
          super().setUp()
          _UpperCAmelCase					:  Optional[Any]         =    True

          _UpperCAmelCase					:  int         =    ["""robot-test/dummy-tokenizer-fast""", """robot-test/dummy-tokenizer-wordlevel"""]
          _UpperCAmelCase					:  Union[str, Any]         =    """SaulLu/dummy-tokenizer-bytelevel-bpe"""

          # Inclusion of 2 tokenizers to test different types of models (Unigram and WordLevel for the moment)
          _UpperCAmelCase					:  List[str]         =    [(PreTrainedTokenizerFast, model_path, {}) for model_path in model_paths]

          _UpperCAmelCase					:  Optional[Any]         =    PreTrainedTokenizerFast.from_pretrained(model_paths[0]	)
          tokenizer.save_pretrained(self.tmpdirname	)



     def     snake_case_ (self	):
          # We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
          # model
          pass



     def     snake_case_ (self	):
          # We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
          # model
          pass



     def     snake_case_ (self	):
          # We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
          # model
          pass



     def     snake_case_ (self	):
          # PreTrainedTokenizerFast doesn't have tokenizer_file in its signature
          pass



     def     snake_case_ (self	):
          _UpperCAmelCase					:  Optional[Any]         =    self.tmpdirname
          # Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
          for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
               with self.subTest(F"{tokenizer.__class__.__name__} ({pretrained_name})"	):
                    try:
                         _UpperCAmelCase					:  Optional[int]         =    tempfile.mkdtemp()
                         _UpperCAmelCase					:  Tuple         =    self.rust_tokenizer_class.from_pretrained(lowerCAmelCase__						,	**lowerCAmelCase__	)

                         tokenizer.save_pretrained(self.tmpdirname	)
                         super().test_training_new_tokenizer()
                    finally:
                         # Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
                         # is restored
                         shutil.rmtree(self.tmpdirname	)
                         _UpperCAmelCase					:  Optional[int]         =    tmpdirname_orig



     def     snake_case_ (self	):
          _UpperCAmelCase					:  int         =    self.tmpdirname
          # Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
          for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
               with self.subTest(F"{tokenizer.__class__.__name__} ({pretrained_name})"	):
                    try:
                         _UpperCAmelCase					:  Any         =    tempfile.mkdtemp()
                         _UpperCAmelCase					:  Union[str, Any]         =    self.rust_tokenizer_class.from_pretrained(lowerCAmelCase__						,	**lowerCAmelCase__	)

                         tokenizer.save_pretrained(self.tmpdirname	)
                         super().test_training_new_tokenizer_with_special_tokens_change()
                    finally:
                         # Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
                         # is restored
                         shutil.rmtree(self.tmpdirname	)
                         _UpperCAmelCase					:  Dict         =    tmpdirname_orig



     def     snake_case_ (self	):
          _UpperCAmelCase					:  List[Any]         =    self.rust_tokenizer_class.from_pretrained(self.bytelevel_bpe_model_name	)

          _UpperCAmelCase					:  Dict         =    ("""a""" for _ in range(1_0_0_0	))
          _UpperCAmelCase					:  Optional[int]         =    tokenizer.train_new_from_iterator(text_iterator=lowerCAmelCase__						,	length=1_0_0_0						,	vocab_size=5_0	)

          _UpperCAmelCase					:  int         =    new_tokenizer.encode("""a🤗"""	)
          self.assertEqual(lowerCAmelCase__						,	[6_4, 1_7_2, 2_5_3, 9_7, 2_4_5]	)







@require_tokenizers
class 	__lowerCAmelCase    (  unittest.TestCase				):



     def     snake_case_ (self	):
          _UpperCAmelCase					:  Any         =    AutoTokenizer.from_pretrained("""bert-base-cased"""	)
          _UpperCAmelCase					:  Dict         =    json.loads(tokenizer._tokenizer.to_str()	)
          _UpperCAmelCase					:  List[Any]         =    len(lowerCAmelCase__	)

          with tempfile.TemporaryDirectory() as tmp_dir:
               # Hack to save this in the tokenizer_config.json
               _UpperCAmelCase					:  Union[str, Any]         =    ["""tokenizer.4.0.0.json"""]
               tokenizer.save_pretrained(lowerCAmelCase__	)
               json.dump(lowerCAmelCase__						,	open(os.path.join(lowerCAmelCase__						,	"""tokenizer.4.0.0.json"""	)						,	"""w"""	)	)

               # This should pick the new tokenizer file as the version of Transformers is > 4.0.0
               _UpperCAmelCase					:  Optional[int]         =    AutoTokenizer.from_pretrained(lowerCAmelCase__	)
               self.assertEqual(len(lowerCAmelCase__	)						,	len(lowerCAmelCase__	) + 1	)
               _UpperCAmelCase					:  Tuple         =    json.loads(new_tokenizer._tokenizer.to_str()	)
               self.assertIn("""huggingface"""						,	json_tokenizer["""model"""]["""vocab"""]	)

               # Will need to be adjusted if we reach v42 and this test is still here.
               # Should pick the old tokenizer file as the version of Transformers is < 4.0.0
               shutil.move(os.path.join(lowerCAmelCase__						,	"""tokenizer.4.0.0.json"""	)						,	os.path.join(lowerCAmelCase__						,	"""tokenizer.42.0.0.json"""	)	)
               _UpperCAmelCase					:  Tuple         =    ["""tokenizer.42.0.0.json"""]
               tokenizer.save_pretrained(lowerCAmelCase__	)
               _UpperCAmelCase					:  str         =    AutoTokenizer.from_pretrained(lowerCAmelCase__	)
               self.assertEqual(len(lowerCAmelCase__	)						,	len(lowerCAmelCase__	)	)
               _UpperCAmelCase					:  Union[str, Any]         =    json.loads(new_tokenizer._tokenizer.to_str()	)
               self.assertNotIn("""huggingface"""						,	json_tokenizer["""model"""]["""vocab"""]	)



     def     snake_case_ (self	):
          # This repo has two tokenizer files, one for v4.0.0 and above with an added token, one for versions lower.
          _UpperCAmelCase					:  List[Any]         =    """hf-internal-testing/test-two-tokenizers"""

          # This should pick the new tokenizer file as the version of Transformers is > 4.0.0
          _UpperCAmelCase					:  Dict         =    AutoTokenizer.from_pretrained(lowerCAmelCase__	)
          self.assertEqual(len(lowerCAmelCase__	)						,	2_8_9_9_7	)
          _UpperCAmelCase					:  List[str]         =    json.loads(tokenizer._tokenizer.to_str()	)
          self.assertIn("""huggingface"""						,	json_tokenizer["""model"""]["""vocab"""]	)

          # Testing an older version by monkey-patching the version in the module it's used.
          import transformers as old_transformers

          _UpperCAmelCase					:  Optional[Any]         =    """3.0.0"""
          _UpperCAmelCase					:  str         =    old_transformers.models.auto.AutoTokenizer.from_pretrained(lowerCAmelCase__	)
          self.assertEqual(len(lowerCAmelCase__	)						,	2_8_9_9_6	)
          _UpperCAmelCase					:  Optional[int]         =    json.loads(old_tokenizer._tokenizer.to_str()	)
          self.assertNotIn("""huggingface"""						,	json_tokenizer["""model"""]["""vocab"""]	)







@require_tokenizers
class 	__lowerCAmelCase    (  unittest.TestCase				):



     def     snake_case_ (self	):
          # See https://github.com/huggingface/transformers/pull/12550
          # and https://github.com/huggingface/tokenizers/issues/537
          _UpperCAmelCase					:  int         =    PreTrainedTokenizerFast.from_pretrained("""robot-test/dummy-tokenizer-wordlevel"""	)
          _UpperCAmelCase					:  str         =    """The Matrix is a 1999 science fiction action film."""

          with concurrent.futures.ThreadPoolExecutor() as executor:
               _UpperCAmelCase					:  List[Any]         =    [executor.submit(self.fetch						,	lowerCAmelCase__						,	lowerCAmelCase__	) for i in range(1_0	)]
               _UpperCAmelCase					:  Any         =    [future.result() for future in futures]
               self.assertEqual(lowerCAmelCase__						,	[[1, 1_0, 0, 8, 0, 1_8, 0, 0, 0, 2] for i in range(1_0	)]	)



     def     snake_case_ (self						,	lowerCAmelCase__						,	lowerCAmelCase__	):
          return tokenizer.encode(lowerCAmelCase__						,	truncation="""longest_first"""						,	padding="""longest"""	)


