





import unittest

from transformers import SPIECE_UNDERLINE, XLNetTokenizer, XLNetTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


lowercase    : Any    	=  get_tests_dir("""fixtures/test_sentencepiece.model""")

@require_sentencepiece
@require_tokenizers
class  __A(						__UpperCAmelCase						,			unittest.TestCase ):
     __A	  =       XLNetTokenizer
     __A	  =       XLNetTokenizerFast
     __A	  =       True
     __A	  =       True

     def 	_UpperCamelCase     (      self      ):





            """simple docstring"""

            super().setUp()

            # We have a SentencePiece fixture for testing
            _UpperCamelCase              =						XLNetTokenizer(A,       keep_accents=A      )
            tokenizer.sanitize_special_tokens()
            tokenizer.save_pretrained(self.tmpdirname      )

     def 	_UpperCamelCase     (      self      ):





            """simple docstring"""

            _UpperCamelCase              =						'''<s>'''
            _UpperCamelCase              =						1

            self.assertEqual(self.get_tokenizer()._convert_token_to_id(A      ),       A      )
            self.assertEqual(self.get_tokenizer()._convert_id_to_token(A      ),       A      )

     def 	_UpperCamelCase     (      self      ):





            """simple docstring"""

            _UpperCamelCase              =						list(self.get_tokenizer().get_vocab().keys()      )

            self.assertEqual(vocab_keys[0],       '''<unk>'''      )
            self.assertEqual(vocab_keys[1],       '''<s>'''      )
            self.assertEqual(vocab_keys[-1],       '''<eod>'''      )
            self.assertEqual(len(A      ),       1006      )

     def 	_UpperCamelCase     (      self      ):





            """simple docstring"""

            self.assertEqual(self.get_tokenizer().vocab_size,       1000      )

     def 	_UpperCamelCase     (      self      ):





            """simple docstring"""

            _UpperCamelCase              =						XLNetTokenizer(A,       keep_accents=A      )

            _UpperCamelCase              =						tokenizer.tokenize('''This is a test'''      )
            self.assertListEqual(A,       ['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est''']      )

            self.assertListEqual(tokenizer.convert_tokens_to_ids(A      ),       [285, 46, 10, 170, 382]      )

            _UpperCamelCase              =						tokenizer.tokenize('''I was born in 92000, and this is falsé.'''      )
            self.assertListEqual(
                A,       [
                    SPIECE_UNDERLINE + '''I''',
                    SPIECE_UNDERLINE + '''was''',
                    SPIECE_UNDERLINE + '''b''',
                    '''or''',
                    '''n''',
                    SPIECE_UNDERLINE + '''in''',
                    SPIECE_UNDERLINE + '''''',
                    '''9''',
                    '''2''',
                    '''0''',
                    '''0''',
                    '''0''',
                    ''',''',
                    SPIECE_UNDERLINE + '''and''',
                    SPIECE_UNDERLINE + '''this''',
                    SPIECE_UNDERLINE + '''is''',
                    SPIECE_UNDERLINE + '''f''',
                    '''al''',
                    '''s''',
                    '''é''',
                    '''.''',
                ],       )
            _UpperCamelCase              =						tokenizer.convert_tokens_to_ids(A      )
            self.assertListEqual(A,       [8, 21, 84, 55, 24, 19, 7, 0, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 0, 4]      )

            _UpperCamelCase              =						tokenizer.convert_ids_to_tokens(A      )
            self.assertListEqual(
                A,       [
                    SPIECE_UNDERLINE + '''I''',
                    SPIECE_UNDERLINE + '''was''',
                    SPIECE_UNDERLINE + '''b''',
                    '''or''',
                    '''n''',
                    SPIECE_UNDERLINE + '''in''',
                    SPIECE_UNDERLINE + '''''',
                    '''<unk>''',
                    '''2''',
                    '''0''',
                    '''0''',
                    '''0''',
                    ''',''',
                    SPIECE_UNDERLINE + '''and''',
                    SPIECE_UNDERLINE + '''this''',
                    SPIECE_UNDERLINE + '''is''',
                    SPIECE_UNDERLINE + '''f''',
                    '''al''',
                    '''s''',
                    '''<unk>''',
                    '''.''',
                ],       )

     def 	_UpperCamelCase     (      self      ):





            """simple docstring"""

            _UpperCamelCase              =						XLNetTokenizer(A,       do_lower_case=A      )
            _UpperCamelCase              =						tokenizer.tokenize('''I was born in 92000, and this is falsé.'''      )
            self.assertListEqual(
                A,       [
                    SPIECE_UNDERLINE + '''''',
                    '''i''',
                    SPIECE_UNDERLINE + '''was''',
                    SPIECE_UNDERLINE + '''b''',
                    '''or''',
                    '''n''',
                    SPIECE_UNDERLINE + '''in''',
                    SPIECE_UNDERLINE + '''''',
                    '''9''',
                    '''2''',
                    '''0''',
                    '''0''',
                    '''0''',
                    ''',''',
                    SPIECE_UNDERLINE + '''and''',
                    SPIECE_UNDERLINE + '''this''',
                    SPIECE_UNDERLINE + '''is''',
                    SPIECE_UNDERLINE + '''f''',
                    '''al''',
                    '''se''',
                    '''.''',
                ],       )
            self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''      ),       ['''▁he''', '''ll''', '''o''']      )

     def 	_UpperCamelCase     (      self      ):





            """simple docstring"""

            _UpperCamelCase              =						XLNetTokenizer(A,       do_lower_case=A      )
            _UpperCamelCase              =						tokenizer.tokenize('''I was born in 92000, and this is falsé.'''      )
            self.assertListEqual(
                A,       [
                    SPIECE_UNDERLINE + '''I''',
                    SPIECE_UNDERLINE + '''was''',
                    SPIECE_UNDERLINE + '''b''',
                    '''or''',
                    '''n''',
                    SPIECE_UNDERLINE + '''in''',
                    SPIECE_UNDERLINE + '''''',
                    '''9''',
                    '''2''',
                    '''0''',
                    '''0''',
                    '''0''',
                    ''',''',
                    SPIECE_UNDERLINE + '''and''',
                    SPIECE_UNDERLINE + '''this''',
                    SPIECE_UNDERLINE + '''is''',
                    SPIECE_UNDERLINE + '''f''',
                    '''al''',
                    '''se''',
                    '''.''',
                ],       )

     @slow
     def 	_UpperCamelCase     (      self      ):





            """simple docstring"""

            _UpperCamelCase              =						XLNetTokenizer.from_pretrained('''xlnet-base-cased'''      )

            _UpperCamelCase              =						tokenizer.encode('''sequence builders''',       add_special_tokens=A      )
            _UpperCamelCase              =						tokenizer.encode('''multi-sequence build''',       add_special_tokens=A      )

            _UpperCamelCase              =						tokenizer.build_inputs_with_special_tokens(A      )
            _UpperCamelCase              =						tokenizer.build_inputs_with_special_tokens(A,       A      )

            assert encoded_sentence == text + [4, 3]
            assert encoded_pair == text + [4] + text_a + [4, 3]



     @slow
     def 	_UpperCamelCase     (      self      ):





            """simple docstring"""

            _UpperCamelCase              =						{'''input_ids''': [[17, 2_1442, 270, 17, 10, 1_4645, 318, 34, 17, 4546, 3145, 787, 13, 7752, 2_2018, 23, 21, 17, 4546, 3145, 787, 13, 3352, 1_4431, 13, 5500, 11, 1176, 580, 13, 1_6819, 4797, 23, 17, 10, 1_7135, 658, 19, 457, 7932, 13, 184, 19, 3154, 1_7135, 6468, 19, 1404, 1_2269, 19, 4229, 5356, 1_6264, 46, 19, 17, 2_0545, 1_0395, 9, 9, 9, 11, 28, 6421, 9531, 2_0729, 17, 10, 353, 1_7022, 11, 21, 6421, 9531, 1_6949, 17, 10, 1_1509, 753, 11, 33, 95, 2421, 7385, 956, 1_4431, 2626, 25, 842, 7385, 4836, 21, 1429, 2272, 9855, 3120, 161, 2_4738, 19, 1_3203, 658, 218, 787, 21, 430, 1_8482, 847, 2637, 9, 4, 3], [5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 322, 2_2178, 27, 1064, 22, 956, 13, 1_1101, 1429, 5854, 2_4313, 1_8953, 40, 422, 2_4366, 68, 1758, 37, 1_0483, 1_4257, 31, 207, 263, 21, 203, 3773, 25, 71, 9735, 9, 4, 3], [5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 32, 2049, 3442, 17, 1_3894, 3380, 23, 95, 18, 1_7634, 2288, 9, 4, 3]], '''token_type_ids''': [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2]], '''attention_mask''': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}  # noqa: E501
            # fmt: on

            self.tokenizer_integration_test_util(
                expected_encoding=A,       model_name='''xlnet-base-cased''',       revision='''c841166438c31ec7ca9a106dee7bb312b73ae511''',       )













import gc
import unittest

from diffusers import FlaxDPMSolverMultistepScheduler, FlaxStableDiffusionPipeline
from diffusers.utils import is_flax_available, slow
from diffusers.utils.testing_utils import require_flax


if is_flax_available():
       import jax
       import jax.numpy as jnp
       from flax.jax_utils import replicate
       from flax.training.common_utils import shard

@slow
@require_flax
class  __A(						unittest.TestCase ):

     def 	_UpperCamelCase     (      self      ):





            """simple docstring"""

            super().tearDown()
            gc.collect()

     def 	_UpperCamelCase     (      self      ):





            """simple docstring"""

            _UpperCamelCase       ,		_UpperCamelCase              =						FlaxStableDiffusionPipeline.from_pretrained(
                '''stabilityai/stable-diffusion-2''',       revision='''bf16''',       dtype=jnp.bfloataa,       )

            _UpperCamelCase              =						'''A painting of a squirrel eating a burger'''
            _UpperCamelCase              =						jax.device_count()
            _UpperCamelCase              =						num_samples * [prompt]
            _UpperCamelCase              =						sd_pipe.prepare_inputs(A      )

            _UpperCamelCase              =						replicate(A      )
            _UpperCamelCase              =						shard(A      )

            _UpperCamelCase              =						jax.random.PRNGKey(0      )
            _UpperCamelCase              =						jax.random.split(A,       jax.device_count()      )

            _UpperCamelCase              =						sd_pipe(A,       A,       A,       num_inference_steps=25,       jit=A      )[0]
            assert images.shape == (jax.device_count(), 1, 768, 768, 3)

            _UpperCamelCase              =						images.reshape((images.shape[0] * images.shape[1],) + images.shape[-3:]      )
            _UpperCamelCase              =						images[0, 253:256, 253:256, -1]

            _UpperCamelCase              =						jnp.asarray(jax.device_get(image_slice.flatten()      )      )
            _UpperCamelCase              =						jnp.array([0.4_238, 0.4_414, 0.4_395, 0.4_453, 0.4_629, 0.4_590, 0.4_531, 0.45_508, 0.4_512]      )
            print(F'''output_slice: {output_slice}'''      )
            assert jnp.abs(output_slice - expected_slice      ).max() < 1E-2



     def 	_UpperCamelCase     (      self      ):





            """simple docstring"""

            _UpperCamelCase              =						'''stabilityai/stable-diffusion-2'''
            _UpperCamelCase       ,		_UpperCamelCase              =						FlaxDPMSolverMultistepScheduler.from_pretrained(A,       subfolder='''scheduler'''      )
            _UpperCamelCase       ,		_UpperCamelCase              =						FlaxStableDiffusionPipeline.from_pretrained(
                A,       scheduler=A,       revision='''bf16''',       dtype=jnp.bfloataa,       )
            _UpperCamelCase              =						scheduler_params

            _UpperCamelCase              =						'''A painting of a squirrel eating a burger'''
            _UpperCamelCase              =						jax.device_count()
            _UpperCamelCase              =						num_samples * [prompt]
            _UpperCamelCase              =						sd_pipe.prepare_inputs(A      )

            _UpperCamelCase              =						replicate(A      )
            _UpperCamelCase              =						shard(A      )

            _UpperCamelCase              =						jax.random.PRNGKey(0      )
            _UpperCamelCase              =						jax.random.split(A,       jax.device_count()      )

            _UpperCamelCase              =						sd_pipe(A,       A,       A,       num_inference_steps=25,       jit=A      )[0]
            assert images.shape == (jax.device_count(), 1, 768, 768, 3)

            _UpperCamelCase              =						images.reshape((images.shape[0] * images.shape[1],) + images.shape[-3:]      )
            _UpperCamelCase              =						images[0, 253:256, 253:256, -1]

            _UpperCamelCase              =						jnp.asarray(jax.device_get(image_slice.flatten()      )      )
            _UpperCamelCase              =						jnp.array([0.4_336, 0.42_969, 0.4_453, 0.4_199, 0.4_297, 0.4_531, 0.4_434, 0.4_434, 0.4_297]      )
            print(F'''output_slice: {output_slice}'''      )
            assert jnp.abs(output_slice - expected_slice      ).max() < 1E-2






