




'''simple docstring'''



import os
import pickle
import tempfile
import unittest
from typing import Callable, Optional

import numpy as np

from transformers import (
    BatchEncoding,
    BertTokenizer,
    BertTokenizerFast,
    PreTrainedTokenizer,
    PreTrainedTokenizerFast,
    TensorType,
    TokenSpan,
    is_tokenizers_available,
)
from transformers.models.gpta.tokenization_gpta import GPTaTokenizer
from transformers.testing_utils import CaptureStderr, require_flax, require_tf, require_tokenizers, require_torch, slow


if is_tokenizers_available():
     from tokenizers import Tokenizer
     from tokenizers.models import WordPiece





class 			lowerCAmelCase      (       unittest.TestCase):


       def 					lowerCAmelCase					(      self			,     __SCREAMING_SNAKE_CASE     )					->     str:



            '''simple docstring'''




            __snake_case			    =     list(tokenizer_class.max_model_input_sizes.keys()     )
            for model_name in sa_models[:1]:
                 __snake_case			    =     tokenizer_class.from_pretrained(__SCREAMING_SNAKE_CASE     )
                 self.assertIsNotNone(__SCREAMING_SNAKE_CASE     )
                 self.assertIsInstance(__SCREAMING_SNAKE_CASE			,     __SCREAMING_SNAKE_CASE     )
                 self.assertIsInstance(__SCREAMING_SNAKE_CASE			,     __SCREAMING_SNAKE_CASE     )

                 for special_tok in tokenizer.all_special_tokens:
                      self.assertIsInstance(__SCREAMING_SNAKE_CASE			,     __SCREAMING_SNAKE_CASE     )
                      __snake_case			    =     tokenizer.convert_tokens_to_ids(__SCREAMING_SNAKE_CASE     )
                      self.assertIsInstance(__SCREAMING_SNAKE_CASE			,     __SCREAMING_SNAKE_CASE     )


       def 					lowerCAmelCase					(      self			,     __SCREAMING_SNAKE_CASE			,     __SCREAMING_SNAKE_CASE = None     )					->     List[Any]:



            '''simple docstring'''




            __snake_case			    =     pickle.dumps(__SCREAMING_SNAKE_CASE     )
            self.assertIsNotNone(__SCREAMING_SNAKE_CASE     )

            __snake_case			    =     pickle.loads(__SCREAMING_SNAKE_CASE     )

            # Ensure is_fast is correctly restored
            self.assertEqual(be_restored.is_fast			,     be_original.is_fast     )

            # Ensure encodings are potentially correctly restored
            if be_original.is_fast:
                 self.assertIsNotNone(be_restored.encodings     )
            else:
                 self.assertIsNone(be_restored.encodings     )

            # Ensure the keys are the same
            for original_v, restored_v in zip(be_original.values()			,     be_restored.values()     ):
                 if equal_op:
                      self.assertTrue(equal_op(__SCREAMING_SNAKE_CASE			,     __SCREAMING_SNAKE_CASE     )     )
                 else:
                      self.assertEqual(__SCREAMING_SNAKE_CASE			,     __SCREAMING_SNAKE_CASE     )


       @slow
       def 					lowerCAmelCase					(      self     )					->     Union[str, Any]:



            '''simple docstring'''




            self.check_tokenizer_from_pretrained(__SCREAMING_SNAKE_CASE     )


       def 					lowerCAmelCase					(      self     )					->     Tuple:



            '''simple docstring'''




            self.assertEqual(TensorType('''tf'''     )			,     TensorType.TENSORFLOW     )
            self.assertEqual(TensorType('''pt'''     )			,     TensorType.PYTORCH     )
            self.assertEqual(TensorType('''np'''     )			,     TensorType.NUMPY     )


       @require_tokenizers
       def 					lowerCAmelCase					(      self     )					->     List[str]:



            '''simple docstring'''




            import numpy as np

            __snake_case			    =     BertTokenizer.from_pretrained('''bert-base-cased'''     )
            __snake_case			    =     BertTokenizerFast.from_pretrained('''bert-base-cased'''     )

            # Python no tensor
            with self.subTest('''BatchEncoding (Python, return_tensors=None)'''     ):
                 self.assert_dump_and_restore(tokenizer_p('''Small example to encode'''     )     )

            with self.subTest('''BatchEncoding (Python, return_tensors=NUMPY)'''     ):
                 self.assert_dump_and_restore(
                     tokenizer_p('''Small example to encode'''			,     return_tensors=TensorType.NUMPY     )			,     np.array_equal     )

            with self.subTest('''BatchEncoding (Rust, return_tensors=None)'''     ):
                 self.assert_dump_and_restore(tokenizer_r('''Small example to encode'''     )     )

            with self.subTest('''BatchEncoding (Rust, return_tensors=NUMPY)'''     ):
                 self.assert_dump_and_restore(
                     tokenizer_r('''Small example to encode'''			,     return_tensors=TensorType.NUMPY     )			,     np.array_equal     )


       @require_tf
       @require_tokenizers
       def 					lowerCAmelCase					(      self     )					->     Optional[int]:



            '''simple docstring'''




            import tensorflow as tf

            def tf_array_equals(__SCREAMING_SNAKE_CASE			,     __SCREAMING_SNAKE_CASE     ):
                 return tf.reduce_all(tf.equal(__SCREAMING_SNAKE_CASE			,     __SCREAMING_SNAKE_CASE     )     )

            __snake_case			    =     BertTokenizer.from_pretrained('''bert-base-cased'''     )
            __snake_case			    =     BertTokenizerFast.from_pretrained('''bert-base-cased'''     )

            with self.subTest('''BatchEncoding (Python, return_tensors=TENSORFLOW)'''     ):
                 self.assert_dump_and_restore(
                     tokenizer_p('''Small example to encode'''			,     return_tensors=TensorType.TENSORFLOW     )			,     __SCREAMING_SNAKE_CASE     )

            with self.subTest('''BatchEncoding (Rust, return_tensors=TENSORFLOW)'''     ):
                 self.assert_dump_and_restore(
                     tokenizer_r('''Small example to encode'''			,     return_tensors=TensorType.TENSORFLOW     )			,     __SCREAMING_SNAKE_CASE     )


       @require_torch
       @require_tokenizers
       def 					lowerCAmelCase					(      self     )					->     Optional[Any]:



            '''simple docstring'''




            import torch

            __snake_case			    =     BertTokenizer.from_pretrained('''bert-base-cased'''     )
            __snake_case			    =     BertTokenizerFast.from_pretrained('''bert-base-cased'''     )

            with self.subTest('''BatchEncoding (Python, return_tensors=PYTORCH)'''     ):
                 self.assert_dump_and_restore(
                     tokenizer_p('''Small example to encode'''			,     return_tensors=TensorType.PYTORCH     )			,     torch.equal     )

            with self.subTest('''BatchEncoding (Rust, return_tensors=PYTORCH)'''     ):
                 self.assert_dump_and_restore(
                     tokenizer_r('''Small example to encode'''			,     return_tensors=TensorType.PYTORCH     )			,     torch.equal     )


       @require_tokenizers
       def 					lowerCAmelCase					(      self     )					->     int:



            '''simple docstring'''




            __snake_case			    =     BertTokenizer.from_pretrained('''bert-base-cased'''     )
            __snake_case			    =     BertTokenizerFast.from_pretrained('''bert-base-cased'''     )

            with self.subTest('''Python Tokenizer'''     ):
                 self.assertFalse(tokenizer_p('''Small example to_encode'''     ).is_fast     )

            with self.subTest('''Rust Tokenizer'''     ):
                 self.assertTrue(tokenizer_r('''Small example to_encode'''     ).is_fast     )


       @require_tokenizers
       def 					lowerCAmelCase					(      self     )					->     Tuple:



            '''simple docstring'''




            __snake_case			    =     BertTokenizerFast.from_pretrained('''bert-base-cased'''     )
            __snake_case			    =     tokenizer_r(['''Test''', '''\xad''', '''test''']			,     is_split_into_words=__SCREAMING_SNAKE_CASE     )

            self.assertEqual(encoded.word_to_tokens(0     )			,     TokenSpan(start=1			,     end=2     )     )
            self.assertEqual(encoded.word_to_tokens(1     )			,     __SCREAMING_SNAKE_CASE     )
            self.assertEqual(encoded.word_to_tokens(2     )			,     TokenSpan(start=2			,     end=3     )     )


       def 					lowerCAmelCase					(      self     )					->     Dict:



            '''simple docstring'''




            __snake_case			    =     BatchEncoding({'''inputs''': [[1, 2, 3], [4, 5, 6]], '''labels''': [0, 1]}     )
            __snake_case			    =     batch.convert_to_tensors(tensor_type='''np'''     )
            self.assertEqual(tensor_batch['''inputs'''].shape			,     (2, 3)     )
            self.assertEqual(tensor_batch['''labels'''].shape			,     (2,)     )
            # test converting the converted
            with CaptureStderr() as cs:
                 __snake_case			    =     batch.convert_to_tensors(tensor_type='''np'''     )
            self.assertFalse(len(cs.err     )			,     msg=F'''should have no warning, but got {cs.err}'''     )

            __snake_case			    =     BatchEncoding({'''inputs''': [1, 2, 3], '''labels''': 0}     )
            __snake_case			    =     batch.convert_to_tensors(tensor_type='''np'''			,     prepend_batch_axis=__SCREAMING_SNAKE_CASE     )
            self.assertEqual(tensor_batch['''inputs'''].shape			,     (1, 3)     )
            self.assertEqual(tensor_batch['''labels'''].shape			,     (1,)     )


       @require_torch
       def 					lowerCAmelCase					(      self     )					->     str:



            '''simple docstring'''




            __snake_case			    =     BatchEncoding({'''inputs''': [[1, 2, 3], [4, 5, 6]], '''labels''': [0, 1]}     )
            __snake_case			    =     batch.convert_to_tensors(tensor_type='''pt'''     )
            self.assertEqual(tensor_batch['''inputs'''].shape			,     (2, 3)     )
            self.assertEqual(tensor_batch['''labels'''].shape			,     (2,)     )
            # test converting the converted
            with CaptureStderr() as cs:
                 __snake_case			    =     batch.convert_to_tensors(tensor_type='''pt'''     )
            self.assertFalse(len(cs.err     )			,     msg=F'''should have no warning, but got {cs.err}'''     )

            __snake_case			    =     BatchEncoding({'''inputs''': [1, 2, 3], '''labels''': 0}     )
            __snake_case			    =     batch.convert_to_tensors(tensor_type='''pt'''			,     prepend_batch_axis=__SCREAMING_SNAKE_CASE     )
            self.assertEqual(tensor_batch['''inputs'''].shape			,     (1, 3)     )
            self.assertEqual(tensor_batch['''labels'''].shape			,     (1,)     )


       @require_tf
       def 					lowerCAmelCase					(      self     )					->     Any:



            '''simple docstring'''




            __snake_case			    =     BatchEncoding({'''inputs''': [[1, 2, 3], [4, 5, 6]], '''labels''': [0, 1]}     )
            __snake_case			    =     batch.convert_to_tensors(tensor_type='''tf'''     )
            self.assertEqual(tensor_batch['''inputs'''].shape			,     (2, 3)     )
            self.assertEqual(tensor_batch['''labels'''].shape			,     (2,)     )
            # test converting the converted
            with CaptureStderr() as cs:
                 __snake_case			    =     batch.convert_to_tensors(tensor_type='''tf'''     )
            self.assertFalse(len(cs.err     )			,     msg=F'''should have no warning, but got {cs.err}'''     )

            __snake_case			    =     BatchEncoding({'''inputs''': [1, 2, 3], '''labels''': 0}     )
            __snake_case			    =     batch.convert_to_tensors(tensor_type='''tf'''			,     prepend_batch_axis=__SCREAMING_SNAKE_CASE     )
            self.assertEqual(tensor_batch['''inputs'''].shape			,     (1, 3)     )
            self.assertEqual(tensor_batch['''labels'''].shape			,     (1,)     )


       @require_flax
       def 					lowerCAmelCase					(      self     )					->     List[Any]:



            '''simple docstring'''




            __snake_case			    =     BatchEncoding({'''inputs''': [[1, 2, 3], [4, 5, 6]], '''labels''': [0, 1]}     )
            __snake_case			    =     batch.convert_to_tensors(tensor_type='''jax'''     )
            self.assertEqual(tensor_batch['''inputs'''].shape			,     (2, 3)     )
            self.assertEqual(tensor_batch['''labels'''].shape			,     (2,)     )
            # test converting the converted
            with CaptureStderr() as cs:
                 __snake_case			    =     batch.convert_to_tensors(tensor_type='''jax'''     )
            self.assertFalse(len(cs.err     )			,     msg=F'''should have no warning, but got {cs.err}'''     )

            __snake_case			    =     BatchEncoding({'''inputs''': [1, 2, 3], '''labels''': 0}     )
            __snake_case			    =     batch.convert_to_tensors(tensor_type='''jax'''			,     prepend_batch_axis=__SCREAMING_SNAKE_CASE     )
            self.assertEqual(tensor_batch['''inputs'''].shape			,     (1, 3)     )
            self.assertEqual(tensor_batch['''labels'''].shape			,     (1,)     )


       def 					lowerCAmelCase					(      self     )					->     Optional[Any]:



            '''simple docstring'''




            __snake_case			    =     [{'''input_ids''': np.array([0, 1, 2]     )}, {'''input_ids''': np.array([0, 1, 2, 3]     )}]
            __snake_case			    =     BertTokenizer.from_pretrained('''bert-base-cased'''     )

            __snake_case			    =     tokenizer.pad(__SCREAMING_SNAKE_CASE			,     padding=__SCREAMING_SNAKE_CASE     )
            self.assertTrue(isinstance(batch['''input_ids''']			,     np.ndarray     )     )
            self.assertEqual(batch['''input_ids'''].tolist()			,     [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]     )
            __snake_case			    =     tokenizer.pad(__SCREAMING_SNAKE_CASE			,     padding=__SCREAMING_SNAKE_CASE			,     return_tensors='''np'''     )
            self.assertTrue(isinstance(batch['''input_ids''']			,     np.ndarray     )     )
            self.assertEqual(batch['''input_ids'''].tolist()			,     [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]     )


       @require_torch
       def 					lowerCAmelCase					(      self     )					->     List[str]:



            '''simple docstring'''




            import torch

            __snake_case			    =     [{'''input_ids''': torch.tensor([0, 1, 2]     )}, {'''input_ids''': torch.tensor([0, 1, 2, 3]     )}]
            __snake_case			    =     BertTokenizer.from_pretrained('''bert-base-cased'''     )

            __snake_case			    =     tokenizer.pad(__SCREAMING_SNAKE_CASE			,     padding=__SCREAMING_SNAKE_CASE     )
            self.assertTrue(isinstance(batch['''input_ids''']			,     torch.Tensor     )     )
            self.assertEqual(batch['''input_ids'''].tolist()			,     [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]     )
            __snake_case			    =     tokenizer.pad(__SCREAMING_SNAKE_CASE			,     padding=__SCREAMING_SNAKE_CASE			,     return_tensors='''pt'''     )
            self.assertTrue(isinstance(batch['''input_ids''']			,     torch.Tensor     )     )
            self.assertEqual(batch['''input_ids'''].tolist()			,     [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]     )


       @require_tf
       def 					lowerCAmelCase					(      self     )					->     Union[str, Any]:



            '''simple docstring'''




            import tensorflow as tf

            __snake_case			    =     [{'''input_ids''': tf.constant([0, 1, 2]     )}, {'''input_ids''': tf.constant([0, 1, 2, 3]     )}]
            __snake_case			    =     BertTokenizer.from_pretrained('''bert-base-cased'''     )

            __snake_case			    =     tokenizer.pad(__SCREAMING_SNAKE_CASE			,     padding=__SCREAMING_SNAKE_CASE     )
            self.assertTrue(isinstance(batch['''input_ids''']			,     tf.Tensor     )     )
            self.assertEqual(batch['''input_ids'''].numpy().tolist()			,     [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]     )
            __snake_case			    =     tokenizer.pad(__SCREAMING_SNAKE_CASE			,     padding=__SCREAMING_SNAKE_CASE			,     return_tensors='''tf'''     )
            self.assertTrue(isinstance(batch['''input_ids''']			,     tf.Tensor     )     )
            self.assertEqual(batch['''input_ids'''].numpy().tolist()			,     [[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]     )


       @require_tokenizers
       def 					lowerCAmelCase					(      self     )					->     Union[str, Any]:



            '''simple docstring'''




            __snake_case			    =     Tokenizer(WordPiece(unk_token='''[UNK]'''     )     )
            PreTrainedTokenizerFast(tokenizer_object=__SCREAMING_SNAKE_CASE     )




       @require_tokenizers
       def 					lowerCAmelCase					(      self     )					->     Optional[Any]:



            '''simple docstring'''




            __snake_case			    =     Tokenizer(WordPiece(unk_token='''[UNK]'''     )     )
            with tempfile.TemporaryDirectory() as tmpdirname:
                 bert_tokenizer.save(os.path.join(__SCREAMING_SNAKE_CASE			,     '''tokenizer.json'''     )     )
                 PreTrainedTokenizerFast(tokenizer_file=os.path.join(__SCREAMING_SNAKE_CASE			,     '''tokenizer.json'''     )     )












'''simple docstring'''



import os

from packaging import version

from .. import __version__
from .accelerate_utils import apply_forward_hook
from .constants import (
    CONFIG_NAME,
    DEPRECATED_REVISION_ARGS,
    DIFFUSERS_CACHE,
    DIFFUSERS_DYNAMIC_MODULE_NAME,
    FLAX_WEIGHTS_NAME,
    HF_MODULES_CACHE,
    HUGGINGFACE_CO_RESOLVE_ENDPOINT,
    ONNX_EXTERNAL_WEIGHTS_NAME,
    ONNX_WEIGHTS_NAME,
    SAFETENSORS_WEIGHTS_NAME,
    TEXT_ENCODER_ATTN_MODULE,
    WEIGHTS_NAME,
)
from .deprecation_utils import deprecate
from .doc_utils import replace_example_docstring
from .dynamic_modules_utils import get_class_from_dynamic_module
from .hub_utils import (
    HF_HUB_OFFLINE,
    _add_variant,
    _get_model_file,
    extract_commit_hash,
    http_user_agent,
)
from .import_utils import (
    BACKENDS_MAPPING,
    ENV_VARS_TRUE_AND_AUTO_VALUES,
    ENV_VARS_TRUE_VALUES,
    USE_JAX,
    USE_TF,
    USE_TORCH,
    DummyObject,
    OptionalDependencyNotAvailable,
    is_accelerate_available,
    is_accelerate_version,
    is_bsa_available,
    is_flax_available,
    is_ftfy_available,
    is_inflect_available,
    is_invisible_watermark_available,
    is_k_diffusion_available,
    is_k_diffusion_version,
    is_librosa_available,
    is_note_seq_available,
    is_omegaconf_available,
    is_onnx_available,
    is_safetensors_available,
    is_scipy_available,
    is_tensorboard_available,
    is_tf_available,
    is_torch_available,
    is_torch_version,
    is_torchsde_available,
    is_transformers_available,
    is_transformers_version,
    is_unidecode_available,
    is_wandb_available,
    is_xformers_available,
    requires_backends,
)
from .logging import get_logger
from .outputs import BaseOutput
from .pil_utils import PIL_INTERPOLATION, numpy_to_pil, pt_to_pil
from .torch_utils import is_compiled_module, randn_tensor


if is_torch_available():
     from .testing_utils import (
         floats_tensor,
         load_hf_numpy,
         load_image,
         load_numpy,
         load_pt,
         nightly,
         parse_flag_from_env,
         print_tensor_test,
         require_torch_a,
         require_torch_gpu,
         skip_mps,
         slow,
         torch_all_close,
         torch_device,
     )
     from .torch_utils import maybe_allow_in_graph

from .testing_utils import export_to_gif, export_to_video


UpperCAmelCase_				:     int     			=						get_logger(__name__)







def 	_UpperCamelCase       (_lowerCamelCase							:   Union[str, Any]		)->      Tuple:




     '''simple docstring'''





     if version.parse(_lowerCamelCase		) < version.parse(_lowerCamelCase		):
          if "dev" in min_version:
               __snake_case			    =     (
                   '''This example requires a source install from HuggingFace diffusers (see '''
                   '''`https://huggingface.co/docs/diffusers/installation#install-from-source`),'''
               )
          else:
               __snake_case			    =     f'''This example requires a minimum version of {min_version},'''
          error_message += f''' but the version found is {__version__}.\n'''
          raise ImportError(_lowerCamelCase		)






