




import torch

from diffusers import HeunDiscreteScheduler
from diffusers.utils import torch_device

from .test_schedulers import SchedulerCommonTest




class 			a__       (					__a      ):
     a		:			List[str]							=	(HeunDiscreteScheduler,)
     a		:			Optional[Any]							=	10







     def 			lowerCAmelCase_				(	self  ,		**A )  ->    str:



          '''simple docstring'''



          a      =      {
              """num_train_timesteps""": 1100,
              """beta_start""": 0.0_0_0_1,
              """beta_end""": 0.0_2,
              """beta_schedule""": """linear""",
          }

          config.update(**lowerCAmelCase__ )
          return config







     def 			lowerCAmelCase_				(	self )  ->    Optional[Any]:



          '''simple docstring'''



          for timesteps in [10, 50, 100, 1000]:
               self.check_over_configs(num_train_timesteps=lowerCAmelCase__ )







     def 			lowerCAmelCase_				(	self )  ->    Union[str, Any]:



          '''simple docstring'''



          for beta_start, beta_end in zip([0.0_0_0_0_1, 0.0_0_0_1, 0.0_0_1]  ,		[0.0_0_0_2, 0.0_0_2, 0.0_2] ):
               self.check_over_configs(beta_start=lowerCAmelCase__  ,		beta_end=lowerCAmelCase__ )







     def 			lowerCAmelCase_				(	self )  ->    int:



          '''simple docstring'''



          for schedule in ["linear", "scaled_linear", "exp"]:
               self.check_over_configs(beta_schedule=lowerCAmelCase__ )







     def 			lowerCAmelCase_				(	self )  ->    Dict:



          '''simple docstring'''



          for clip_sample_range in [1.0, 2.0, 3.0]:
               self.check_over_configs(clip_sample_range=lowerCAmelCase__  ,		clip_sample=lowerCAmelCase__ )







     def 			lowerCAmelCase_				(	self )  ->    int:



          '''simple docstring'''



          for prediction_type in ["epsilon", "v_prediction", "sample"]:
               self.check_over_configs(prediction_type=lowerCAmelCase__ )







     def 			lowerCAmelCase_				(	self )  ->    Tuple:



          '''simple docstring'''



          a      =      self.scheduler_classes[0]
          a      =      self.get_scheduler_config()
          a      =      scheduler_class(**lowerCAmelCase__ )

          scheduler.set_timesteps(self.num_inference_steps )

          a      =      self.dummy_model()
          a      =      self.dummy_sample_deter * scheduler.init_noise_sigma
          a      =      sample.to(lowerCAmelCase__ )

          for i, t in enumerate(scheduler.timesteps ):
               a      =      scheduler.scale_model_input(lowerCAmelCase__  ,		lowerCAmelCase__ )

               a      =      model(lowerCAmelCase__  ,		lowerCAmelCase__ )

               a      =      scheduler.step(lowerCAmelCase__  ,		lowerCAmelCase__  ,		lowerCAmelCase__ )
               a      =      output.prev_sample

          a      =      torch.sum(torch.abs(lowerCAmelCase__ ) )
          a      =      torch.mean(torch.abs(lowerCAmelCase__ ) )

          if torch_device in ["cpu", "mps"]:
               assert abs(result_sum.item() - 0.1_2_3_3 ) < 1e-2
               assert abs(result_mean.item() - 0.0_0_0_2 ) < 1e-3
          else:
               # CUDA
               assert abs(result_sum.item() - 0.1_2_3_3 ) < 1e-2
               assert abs(result_mean.item() - 0.0_0_0_2 ) < 1e-3







     def 			lowerCAmelCase_				(	self )  ->    Any:



          '''simple docstring'''



          a      =      self.scheduler_classes[0]
          a      =      self.get_scheduler_config(prediction_type="v_prediction" )
          a      =      scheduler_class(**lowerCAmelCase__ )

          scheduler.set_timesteps(self.num_inference_steps )

          a      =      self.dummy_model()
          a      =      self.dummy_sample_deter * scheduler.init_noise_sigma
          a      =      sample.to(lowerCAmelCase__ )

          for i, t in enumerate(scheduler.timesteps ):
               a      =      scheduler.scale_model_input(lowerCAmelCase__  ,		lowerCAmelCase__ )

               a      =      model(lowerCAmelCase__  ,		lowerCAmelCase__ )

               a      =      scheduler.step(lowerCAmelCase__  ,		lowerCAmelCase__  ,		lowerCAmelCase__ )
               a      =      output.prev_sample

          a      =      torch.sum(torch.abs(lowerCAmelCase__ ) )
          a      =      torch.mean(torch.abs(lowerCAmelCase__ ) )

          if torch_device in ["cpu", "mps"]:
               assert abs(result_sum.item() - 4.6_934e-07 ) < 1e-2
               assert abs(result_mean.item() - 6.1_112e-10 ) < 1e-3
          else:
               # CUDA
               assert abs(result_sum.item() - 4.693_428_650_170_972e-07 ) < 1e-2
               assert abs(result_mean.item() - 0.0_0_0_2 ) < 1e-3







     def 			lowerCAmelCase_				(	self )  ->    Dict:



          '''simple docstring'''



          a      =      self.scheduler_classes[0]
          a      =      self.get_scheduler_config()
          a      =      scheduler_class(**lowerCAmelCase__ )

          scheduler.set_timesteps(self.num_inference_steps  ,		device=lowerCAmelCase__ )

          a      =      self.dummy_model()
          a      =      self.dummy_sample_deter.to(lowerCAmelCase__ ) * scheduler.init_noise_sigma

          for t in scheduler.timesteps:
               a      =      scheduler.scale_model_input(lowerCAmelCase__  ,		lowerCAmelCase__ )

               a      =      model(lowerCAmelCase__  ,		lowerCAmelCase__ )

               a      =      scheduler.step(lowerCAmelCase__  ,		lowerCAmelCase__  ,		lowerCAmelCase__ )
               a      =      output.prev_sample

          a      =      torch.sum(torch.abs(lowerCAmelCase__ ) )
          a      =      torch.mean(torch.abs(lowerCAmelCase__ ) )

          if str(lowerCAmelCase__ ).startswith("cpu" ):
               # The following sum varies between 148 and 156 on mps. Why?
               assert abs(result_sum.item() - 0.1_2_3_3 ) < 1e-2
               assert abs(result_mean.item() - 0.0_0_0_2 ) < 1e-3
          elif str(lowerCAmelCase__ ).startswith("mps" ):
               # Larger tolerance on mps
               assert abs(result_mean.item() - 0.0_0_0_2 ) < 1e-2
          else:
               # CUDA
               assert abs(result_sum.item() - 0.1_2_3_3 ) < 1e-2
               assert abs(result_mean.item() - 0.0_0_0_2 ) < 1e-3







     def 			lowerCAmelCase_				(	self )  ->    str:



          '''simple docstring'''



          a      =      self.scheduler_classes[0]
          a      =      self.get_scheduler_config()
          a      =      scheduler_class(**lowerCAmelCase__  ,		use_karras_sigmas=lowerCAmelCase__ )

          scheduler.set_timesteps(self.num_inference_steps  ,		device=lowerCAmelCase__ )

          a      =      self.dummy_model()
          a      =      self.dummy_sample_deter.to(lowerCAmelCase__ ) * scheduler.init_noise_sigma
          a      =      sample.to(lowerCAmelCase__ )

          for t in scheduler.timesteps:
               a      =      scheduler.scale_model_input(lowerCAmelCase__  ,		lowerCAmelCase__ )

               a      =      model(lowerCAmelCase__  ,		lowerCAmelCase__ )

               a      =      scheduler.step(lowerCAmelCase__  ,		lowerCAmelCase__  ,		lowerCAmelCase__ )
               a      =      output.prev_sample

          a      =      torch.sum(torch.abs(lowerCAmelCase__ ) )
          a      =      torch.mean(torch.abs(lowerCAmelCase__ ) )

          assert abs(result_sum.item() - 0.0_0_0_1_5 ) < 1e-2
          assert abs(result_mean.item() - 1.9_869_554_535_034_695e-07 ) < 1e-2



'''simple docstring'''

import concurrent.futures
import json
import os
import shutil
import tempfile
import unittest

from transformers import AutoTokenizer, PreTrainedTokenizerFast
from transformers.testing_utils import require_tokenizers

from ..test_tokenization_common import TokenizerTesterMixin







@require_tokenizers
class 	__lowerCAmelCase    (  __a       ,					unittest.TestCase				):
     snake_case   :    Dict							      =      PreTrainedTokenizerFast
     snake_case   :    Optional[Any]							      =      False
     snake_case   :    List[str]							      =      True
     snake_case   :    Any							      =      """tokenizer_file"""



     def     snake_case_ (self	):
          _UpperCAmelCase					:  Dict         =    False  # because we don't have pretrained_vocab_files_map
          super().setUp()
          _UpperCAmelCase					:  Optional[Any]         =    True

          _UpperCAmelCase					:  int         =    ["""robot-test/dummy-tokenizer-fast""", """robot-test/dummy-tokenizer-wordlevel"""]
          _UpperCAmelCase					:  Union[str, Any]         =    """SaulLu/dummy-tokenizer-bytelevel-bpe"""

          # Inclusion of 2 tokenizers to test different types of models (Unigram and WordLevel for the moment)
          _UpperCAmelCase					:  List[str]         =    [(PreTrainedTokenizerFast, model_path, {}) for model_path in model_paths]

          _UpperCAmelCase					:  Optional[Any]         =    PreTrainedTokenizerFast.from_pretrained(model_paths[0]	)
          tokenizer.save_pretrained(self.tmpdirname	)



     def     snake_case_ (self	):
          # We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
          # model
          pass



     def     snake_case_ (self	):
          # We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
          # model
          pass



     def     snake_case_ (self	):
          # We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
          # model
          pass



     def     snake_case_ (self	):
          # PreTrainedTokenizerFast doesn't have tokenizer_file in its signature
          pass



     def     snake_case_ (self	):
          _UpperCAmelCase					:  Optional[Any]         =    self.tmpdirname
          # Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
          for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
               with self.subTest(F"{tokenizer.__class__.__name__} ({pretrained_name})"	):
                    try:
                         _UpperCAmelCase					:  Optional[int]         =    tempfile.mkdtemp()
                         _UpperCAmelCase					:  Tuple         =    self.rust_tokenizer_class.from_pretrained(lowerCAmelCase__						,	**lowerCAmelCase__	)

                         tokenizer.save_pretrained(self.tmpdirname	)
                         super().test_training_new_tokenizer()
                    finally:
                         # Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
                         # is restored
                         shutil.rmtree(self.tmpdirname	)
                         _UpperCAmelCase					:  Optional[int]         =    tmpdirname_orig



     def     snake_case_ (self	):
          _UpperCAmelCase					:  int         =    self.tmpdirname
          # Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
          for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
               with self.subTest(F"{tokenizer.__class__.__name__} ({pretrained_name})"	):
                    try:
                         _UpperCAmelCase					:  Any         =    tempfile.mkdtemp()
                         _UpperCAmelCase					:  Union[str, Any]         =    self.rust_tokenizer_class.from_pretrained(lowerCAmelCase__						,	**lowerCAmelCase__	)

                         tokenizer.save_pretrained(self.tmpdirname	)
                         super().test_training_new_tokenizer_with_special_tokens_change()
                    finally:
                         # Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
                         # is restored
                         shutil.rmtree(self.tmpdirname	)
                         _UpperCAmelCase					:  Dict         =    tmpdirname_orig



     def     snake_case_ (self	):
          _UpperCAmelCase					:  List[Any]         =    self.rust_tokenizer_class.from_pretrained(self.bytelevel_bpe_model_name	)

          _UpperCAmelCase					:  Dict         =    ("""a""" for _ in range(1_0_0_0	))
          _UpperCAmelCase					:  Optional[int]         =    tokenizer.train_new_from_iterator(text_iterator=lowerCAmelCase__						,	length=1_0_0_0						,	vocab_size=5_0	)

          _UpperCAmelCase					:  int         =    new_tokenizer.encode("""a🤗"""	)
          self.assertEqual(lowerCAmelCase__						,	[6_4, 1_7_2, 2_5_3, 9_7, 2_4_5]	)







@require_tokenizers
class 	__lowerCAmelCase    (  unittest.TestCase				):



     def     snake_case_ (self	):
          _UpperCAmelCase					:  Any         =    AutoTokenizer.from_pretrained("""bert-base-cased"""	)
          _UpperCAmelCase					:  Dict         =    json.loads(tokenizer._tokenizer.to_str()	)
          _UpperCAmelCase					:  List[Any]         =    len(lowerCAmelCase__	)

          with tempfile.TemporaryDirectory() as tmp_dir:
               # Hack to save this in the tokenizer_config.json
               _UpperCAmelCase					:  Union[str, Any]         =    ["""tokenizer.4.0.0.json"""]
               tokenizer.save_pretrained(lowerCAmelCase__	)
               json.dump(lowerCAmelCase__						,	open(os.path.join(lowerCAmelCase__						,	"""tokenizer.4.0.0.json"""	)						,	"""w"""	)	)

               # This should pick the new tokenizer file as the version of Transformers is > 4.0.0
               _UpperCAmelCase					:  Optional[int]         =    AutoTokenizer.from_pretrained(lowerCAmelCase__	)
               self.assertEqual(len(lowerCAmelCase__	)						,	len(lowerCAmelCase__	) + 1	)
               _UpperCAmelCase					:  Tuple         =    json.loads(new_tokenizer._tokenizer.to_str()	)
               self.assertIn("""huggingface"""						,	json_tokenizer["""model"""]["""vocab"""]	)

               # Will need to be adjusted if we reach v42 and this test is still here.
               # Should pick the old tokenizer file as the version of Transformers is < 4.0.0
               shutil.move(os.path.join(lowerCAmelCase__						,	"""tokenizer.4.0.0.json"""	)						,	os.path.join(lowerCAmelCase__						,	"""tokenizer.42.0.0.json"""	)	)
               _UpperCAmelCase					:  Tuple         =    ["""tokenizer.42.0.0.json"""]
               tokenizer.save_pretrained(lowerCAmelCase__	)
               _UpperCAmelCase					:  str         =    AutoTokenizer.from_pretrained(lowerCAmelCase__	)
               self.assertEqual(len(lowerCAmelCase__	)						,	len(lowerCAmelCase__	)	)
               _UpperCAmelCase					:  Union[str, Any]         =    json.loads(new_tokenizer._tokenizer.to_str()	)
               self.assertNotIn("""huggingface"""						,	json_tokenizer["""model"""]["""vocab"""]	)



     def     snake_case_ (self	):
          # This repo has two tokenizer files, one for v4.0.0 and above with an added token, one for versions lower.
          _UpperCAmelCase					:  List[Any]         =    """hf-internal-testing/test-two-tokenizers"""

          # This should pick the new tokenizer file as the version of Transformers is > 4.0.0
          _UpperCAmelCase					:  Dict         =    AutoTokenizer.from_pretrained(lowerCAmelCase__	)
          self.assertEqual(len(lowerCAmelCase__	)						,	2_8_9_9_7	)
          _UpperCAmelCase					:  List[str]         =    json.loads(tokenizer._tokenizer.to_str()	)
          self.assertIn("""huggingface"""						,	json_tokenizer["""model"""]["""vocab"""]	)

          # Testing an older version by monkey-patching the version in the module it's used.
          import transformers as old_transformers

          _UpperCAmelCase					:  Optional[Any]         =    """3.0.0"""
          _UpperCAmelCase					:  str         =    old_transformers.models.auto.AutoTokenizer.from_pretrained(lowerCAmelCase__	)
          self.assertEqual(len(lowerCAmelCase__	)						,	2_8_9_9_6	)
          _UpperCAmelCase					:  Optional[int]         =    json.loads(old_tokenizer._tokenizer.to_str()	)
          self.assertNotIn("""huggingface"""						,	json_tokenizer["""model"""]["""vocab"""]	)







@require_tokenizers
class 	__lowerCAmelCase    (  unittest.TestCase				):



     def     snake_case_ (self	):
          # See https://github.com/huggingface/transformers/pull/12550
          # and https://github.com/huggingface/tokenizers/issues/537
          _UpperCAmelCase					:  int         =    PreTrainedTokenizerFast.from_pretrained("""robot-test/dummy-tokenizer-wordlevel"""	)
          _UpperCAmelCase					:  str         =    """The Matrix is a 1999 science fiction action film."""

          with concurrent.futures.ThreadPoolExecutor() as executor:
               _UpperCAmelCase					:  List[Any]         =    [executor.submit(self.fetch						,	lowerCAmelCase__						,	lowerCAmelCase__	) for i in range(1_0	)]
               _UpperCAmelCase					:  Any         =    [future.result() for future in futures]
               self.assertEqual(lowerCAmelCase__						,	[[1, 1_0, 0, 8, 0, 1_8, 0, 0, 0, 2] for i in range(1_0	)]	)



     def     snake_case_ (self						,	lowerCAmelCase__						,	lowerCAmelCase__	):
          return tokenizer.encode(lowerCAmelCase__						,	truncation="""longest_first"""						,	padding="""longest"""	)


