





import unittest

from transformers import SPIECE_UNDERLINE, XLNetTokenizer, XLNetTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


a__      :     int         =		get_tests_dir('fixtures/test_sentencepiece.model')





@require_sentencepiece
@require_tokenizers
class UpperCAmelCase_				( _SCREAMING_SNAKE_CASE     ,		unittest.TestCase ):
  __lowerCAmelCase   :						int							=   XLNetTokenizer
  __lowerCAmelCase   :						Optional[Any]							=   XLNetTokenizerFast
  __lowerCAmelCase   :						Dict							=   True
  __lowerCAmelCase   :						List[str]							=   True


  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       super().setUp()

       # We have a SentencePiece fixture for testing
       A_          =       XLNetTokenizer(__snake_case ,keep_accents=__snake_case )
       tokenizer.sanitize_special_tokens()
       tokenizer.save_pretrained(self.tmpdirname )


  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       A_          =       '''<s>'''
       A_          =       1

       self.assertEqual(self.get_tokenizer()._convert_token_to_id(__snake_case ) ,__snake_case )
       self.assertEqual(self.get_tokenizer()._convert_id_to_token(__snake_case ) ,__snake_case )


  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       A_          =       list(self.get_tokenizer().get_vocab().keys() )

       self.assertEqual(vocab_keys[0] ,'''<unk>''' )
       self.assertEqual(vocab_keys[1] ,'''<s>''' )
       self.assertEqual(vocab_keys[-1] ,'''<eod>''' )
       self.assertEqual(len(__snake_case ) ,1_0_0_6 )


  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       self.assertEqual(self.get_tokenizer().vocab_size ,1_0_0_0 )


  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       A_          =       XLNetTokenizer(__snake_case ,keep_accents=__snake_case )

       A_          =       tokenizer.tokenize('''This is a test''' )
       self.assertListEqual(__snake_case ,['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est'''] )

       self.assertListEqual(tokenizer.convert_tokens_to_ids(__snake_case ) ,[2_8_5, 4_6, 1_0, 1_7_0, 3_8_2] )

       A_          =       tokenizer.tokenize('''I was born in 92000, and this is falsé.''' )
       self.assertListEqual(
           __snake_case ,[
               SPIECE_UNDERLINE + '''I''',
               SPIECE_UNDERLINE + '''was''',
               SPIECE_UNDERLINE + '''b''',
               '''or''',
               '''n''',
               SPIECE_UNDERLINE + '''in''',
               SPIECE_UNDERLINE + '''''',
               '''9''',
               '''2''',
               '''0''',
               '''0''',
               '''0''',
               ''',''',
               SPIECE_UNDERLINE + '''and''',
               SPIECE_UNDERLINE + '''this''',
               SPIECE_UNDERLINE + '''is''',
               SPIECE_UNDERLINE + '''f''',
               '''al''',
               '''s''',
               '''é''',
               '''.''',
           ] ,)
       A_          =       tokenizer.convert_tokens_to_ids(__snake_case )
       self.assertListEqual(__snake_case ,[8, 2_1, 8_4, 5_5, 2_4, 1_9, 7, 0, 6_0_2, 3_4_7, 3_4_7, 3_4_7, 3, 1_2, 6_6, 4_6, 7_2, 8_0, 6, 0, 4] )

       A_          =       tokenizer.convert_ids_to_tokens(__snake_case )
       self.assertListEqual(
           __snake_case ,[
               SPIECE_UNDERLINE + '''I''',
               SPIECE_UNDERLINE + '''was''',
               SPIECE_UNDERLINE + '''b''',
               '''or''',
               '''n''',
               SPIECE_UNDERLINE + '''in''',
               SPIECE_UNDERLINE + '''''',
               '''<unk>''',
               '''2''',
               '''0''',
               '''0''',
               '''0''',
               ''',''',
               SPIECE_UNDERLINE + '''and''',
               SPIECE_UNDERLINE + '''this''',
               SPIECE_UNDERLINE + '''is''',
               SPIECE_UNDERLINE + '''f''',
               '''al''',
               '''s''',
               '''<unk>''',
               '''.''',
           ] ,)


  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       A_          =       XLNetTokenizer(__snake_case ,do_lower_case=__snake_case )
       A_          =       tokenizer.tokenize('''I was born in 92000, and this is falsé.''' )
       self.assertListEqual(
           __snake_case ,[
               SPIECE_UNDERLINE + '''''',
               '''i''',
               SPIECE_UNDERLINE + '''was''',
               SPIECE_UNDERLINE + '''b''',
               '''or''',
               '''n''',
               SPIECE_UNDERLINE + '''in''',
               SPIECE_UNDERLINE + '''''',
               '''9''',
               '''2''',
               '''0''',
               '''0''',
               '''0''',
               ''',''',
               SPIECE_UNDERLINE + '''and''',
               SPIECE_UNDERLINE + '''this''',
               SPIECE_UNDERLINE + '''is''',
               SPIECE_UNDERLINE + '''f''',
               '''al''',
               '''se''',
               '''.''',
           ] ,)
       self.assertListEqual(tokenizer.tokenize('''H\u00E9llo''' ) ,['''▁he''', '''ll''', '''o'''] )


  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       A_          =       XLNetTokenizer(__snake_case ,do_lower_case=__snake_case )
       A_          =       tokenizer.tokenize('''I was born in 92000, and this is falsé.''' )
       self.assertListEqual(
           __snake_case ,[
               SPIECE_UNDERLINE + '''I''',
               SPIECE_UNDERLINE + '''was''',
               SPIECE_UNDERLINE + '''b''',
               '''or''',
               '''n''',
               SPIECE_UNDERLINE + '''in''',
               SPIECE_UNDERLINE + '''''',
               '''9''',
               '''2''',
               '''0''',
               '''0''',
               '''0''',
               ''',''',
               SPIECE_UNDERLINE + '''and''',
               SPIECE_UNDERLINE + '''this''',
               SPIECE_UNDERLINE + '''is''',
               SPIECE_UNDERLINE + '''f''',
               '''al''',
               '''se''',
               '''.''',
           ] ,)


  @slow
  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       A_          =       XLNetTokenizer.from_pretrained('''xlnet-base-cased''' )

       A_          =       tokenizer.encode('''sequence builders''' ,add_special_tokens=__snake_case )
       A_          =       tokenizer.encode('''multi-sequence build''' ,add_special_tokens=__snake_case )

       A_          =       tokenizer.build_inputs_with_special_tokens(__snake_case )
       A_          =       tokenizer.build_inputs_with_special_tokens(__snake_case ,__snake_case )

       assert encoded_sentence == text + [4, 3]
       assert encoded_pair == text + [4] + text_a + [4, 3]




  @slow
  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       A_          =       {'''input_ids''': [[1_7, 2_1_4_4_2, 2_7_0, 1_7, 1_0, 1_4_6_4_5, 3_1_8, 3_4, 1_7, 4_5_4_6, 3_1_4_5, 7_8_7, 1_3, 7_7_5_2, 2_2_0_1_8, 2_3, 2_1, 1_7, 4_5_4_6, 3_1_4_5, 7_8_7, 1_3, 3_3_5_2, 1_4_4_3_1, 1_3, 5_5_0_0, 1_1, 1_1_7_6, 5_8_0, 1_3, 1_6_8_1_9, 4_7_9_7, 2_3, 1_7, 1_0, 1_7_1_3_5, 6_5_8, 1_9, 4_5_7, 7_9_3_2, 1_3, 1_8_4, 1_9, 3_1_5_4, 1_7_1_3_5, 6_4_6_8, 1_9, 1_4_0_4, 1_2_2_6_9, 1_9, 4_2_2_9, 5_3_5_6, 1_6_2_6_4, 4_6, 1_9, 1_7, 2_0_5_4_5, 1_0_3_9_5, 9, 9, 9, 1_1, 2_8, 6_4_2_1, 9_5_3_1, 2_0_7_2_9, 1_7, 1_0, 3_5_3, 1_7_0_2_2, 1_1, 2_1, 6_4_2_1, 9_5_3_1, 1_6_9_4_9, 1_7, 1_0, 1_1_5_0_9, 7_5_3, 1_1, 3_3, 9_5, 2_4_2_1, 7_3_8_5, 9_5_6, 1_4_4_3_1, 2_6_2_6, 2_5, 8_4_2, 7_3_8_5, 4_8_3_6, 2_1, 1_4_2_9, 2_2_7_2, 9_8_5_5, 3_1_2_0, 1_6_1, 2_4_7_3_8, 1_9, 1_3_2_0_3, 6_5_8, 2_1_8, 7_8_7, 2_1, 4_3_0, 1_8_4_8_2, 8_4_7, 2_6_3_7, 9, 4, 3], [5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 3_2_2, 2_2_1_7_8, 2_7, 1_0_6_4, 2_2, 9_5_6, 1_3, 1_1_1_0_1, 1_4_2_9, 5_8_5_4, 2_4_3_1_3, 1_8_9_5_3, 4_0, 4_2_2, 2_4_3_6_6, 6_8, 1_7_5_8, 3_7, 1_0_4_8_3, 1_4_2_5_7, 3_1, 2_0_7, 2_6_3, 2_1, 2_0_3, 3_7_7_3, 2_5, 7_1, 9_7_3_5, 9, 4, 3], [5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 3_2, 2_0_4_9, 3_4_4_2, 1_7, 1_3_8_9_4, 3_3_8_0, 2_3, 9_5, 1_8, 1_7_6_3_4, 2_2_8_8, 9, 4, 3]], '''token_type_ids''': [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2]], '''attention_mask''': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}  # noqa: E501
       # fmt: on

       self.tokenizer_integration_test_util(
           expected_encoding=__snake_case ,model_name='''xlnet-base-cased''' ,revision='''c841166438c31ec7ca9a106dee7bb312b73ae511''' ,)










import torch

from diffusers import KDPMaDiscreteScheduler
from diffusers.utils import torch_device

from .test_schedulers import SchedulerCommonTest





class UpperCAmelCase_				( _SCREAMING_SNAKE_CASE ):
  __lowerCAmelCase   :						int							=   (KDPMaDiscreteScheduler,)
  __lowerCAmelCase   :						Union[str, Any]							=   10


  def 						__UpperCAmelCase     (							self ,**__snake_case ):



       """simple docstring"""





       A_          =       {
           '''num_train_timesteps''': 1_1_0_0,
           '''beta_start''': 0.0001,
           '''beta_end''': 0.02,
           '''beta_schedule''': '''linear''',
       }

       config.update(**__snake_case )
       return config


  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       for timesteps in [1_0, 5_0, 1_0_0, 1_0_0_0]:
            self.check_over_configs(num_train_timesteps=__snake_case )


  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       for beta_start, beta_end in zip([0.0_0001, 0.0001, 0.001] ,[0.0002, 0.002, 0.02] ):
            self.check_over_configs(beta_start=__snake_case ,beta_end=__snake_case )


  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       for schedule in ["linear", "scaled_linear"]:
            self.check_over_configs(beta_schedule=__snake_case )


  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       for prediction_type in ["epsilon", "v_prediction"]:
            self.check_over_configs(prediction_type=__snake_case )


  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       A_          =       self.scheduler_classes[0]
       A_          =       self.get_scheduler_config(prediction_type='''v_prediction''' )
       A_          =       scheduler_class(**__snake_case )

       scheduler.set_timesteps(self.num_inference_steps )

       A_          =       self.dummy_model()
       A_          =       self.dummy_sample_deter * scheduler.init_noise_sigma
       A_          =       sample.to(__snake_case )

       for i, t in enumerate(scheduler.timesteps ):
            A_          =       scheduler.scale_model_input(__snake_case ,__snake_case )

            A_          =       model(__snake_case ,__snake_case )

            A_          =       scheduler.step(__snake_case ,__snake_case ,__snake_case )
            A_          =       output.prev_sample

       A_          =       torch.sum(torch.abs(__snake_case ) )
       A_          =       torch.mean(torch.abs(__snake_case ) )

       if torch_device in ["cpu", "mps"]:
            assert abs(result_sum.item() - 4.69_34E-07 ) < 1E-2
            assert abs(result_mean.item() - 6.11_12E-10 ) < 1E-3
       else:
            # CUDA
            assert abs(result_sum.item() - 4.6_93_42_86_50_17_09_72E-07 ) < 1E-2
            assert abs(result_mean.item() - 0.0002 ) < 1E-3


  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       if torch_device == "mps":
            return
       A_          =       self.scheduler_classes[0]
       A_          =       self.get_scheduler_config()
       A_          =       scheduler_class(**__snake_case )

       scheduler.set_timesteps(self.num_inference_steps )

       A_          =       self.dummy_model()
       A_          =       self.dummy_sample_deter * scheduler.init_noise_sigma
       A_          =       sample.to(__snake_case )

       for i, t in enumerate(scheduler.timesteps ):
            A_          =       scheduler.scale_model_input(__snake_case ,__snake_case )

            A_          =       model(__snake_case ,__snake_case )

            A_          =       scheduler.step(__snake_case ,__snake_case ,__snake_case )
            A_          =       output.prev_sample

       A_          =       torch.sum(torch.abs(__snake_case ) )
       A_          =       torch.mean(torch.abs(__snake_case ) )

       if torch_device in ["cpu", "mps"]:
            assert abs(result_sum.item() - 20.4125 ) < 1E-2
            assert abs(result_mean.item() - 0.0266 ) < 1E-3
       else:
            # CUDA
            assert abs(result_sum.item() - 20.4125 ) < 1E-2
            assert abs(result_mean.item() - 0.0266 ) < 1E-3




  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       if torch_device == "mps":
            return
       A_          =       self.scheduler_classes[0]
       A_          =       self.get_scheduler_config()
       A_          =       scheduler_class(**__snake_case )

       scheduler.set_timesteps(self.num_inference_steps ,device=__snake_case )

       A_          =       self.dummy_model()
       A_          =       self.dummy_sample_deter.to(__snake_case ) * scheduler.init_noise_sigma

       for t in scheduler.timesteps:
            A_          =       scheduler.scale_model_input(__snake_case ,__snake_case )

            A_          =       model(__snake_case ,__snake_case )

            A_          =       scheduler.step(__snake_case ,__snake_case ,__snake_case )
            A_          =       output.prev_sample

       A_          =       torch.sum(torch.abs(__snake_case ) )
       A_          =       torch.mean(torch.abs(__snake_case ) )

       if str(__snake_case ).startswith('''cpu''' ):
            # The following sum varies between 148 and 156 on mps. Why?
            assert abs(result_sum.item() - 20.4125 ) < 1E-2
            assert abs(result_mean.item() - 0.0266 ) < 1E-3
       else:
            # CUDA
            assert abs(result_sum.item() - 20.4125 ) < 1E-2
            assert abs(result_mean.item() - 0.0266 ) < 1E-3



