






import os
import sys
import unittest


SCREAMING_SNAKE_CASE 							=						os.path.abspath(os.path.dirname(os.path.dirname(os.path.dirname(__file__))))
sys.path.append(os.path.join(git_repo_path, 'utils'))

import check_dummies  # noqa: E402
from check_dummies import create_dummy_files, create_dummy_object, find_backend, read_init  # noqa: E402


# Align TRANSFORMERS_PATH in check_dummies with the current path
SCREAMING_SNAKE_CASE 							=						os.path.join(git_repo_path, 'src', 'diffusers')
class 							A_    (						unittest.TestCase			):






 '''simple docstring'''




 def  snake_case__				(	self) ->		Optional[Any]:

     """simple docstring"""



     _UpperCAmelCase				:      int			  =				find_backend('''    if not is_torch_available():''')
     self.assertEqual(_A							,  '''torch''')

     # backend_with_underscore = find_backend("    if not is_tensorflow_text_available():")
     # self.assertEqual(backend_with_underscore, "tensorflow_text")

     _UpperCAmelCase				:      List[str]			  =				find_backend('''    if not (is_torch_available() and is_transformers_available()):''')
     self.assertEqual(_A							,  '''torch_and_transformers''')

     # double_backend_with_underscore = find_backend(
     #    "    if not (is_sentencepiece_available() and is_tensorflow_text_available()):"
     # )
     # self.assertEqual(double_backend_with_underscore, "sentencepiece_and_tensorflow_text")

     _UpperCAmelCase				:      List[str]			  =				find_backend(
         '''    if not (is_torch_available() and is_transformers_available() and is_onnx_available()):''')
     self.assertEqual(_A							,  '''torch_and_transformers_and_onnx''')


 def  snake_case__				(	self) ->		List[str]:

     """simple docstring"""



     _UpperCAmelCase				:      Optional[int]			  =				read_init()
     # We don't assert on the exact list of keys to allow for smooth grow of backend-specific objects
     self.assertIn('''torch'''							,  _A)
     self.assertIn('''torch_and_transformers'''							,  _A)
     self.assertIn('''flax_and_transformers'''							,  _A)
     self.assertIn('''torch_and_transformers_and_onnx'''							,  _A)

     # Likewise, we can't assert on the exact content of a key
     self.assertIn('''UNet2DModel'''							,  objects['''torch'''])
     self.assertIn('''FlaxUNet2DConditionModel'''							,  objects['''flax'''])
     self.assertIn('''StableDiffusionPipeline'''							,  objects['''torch_and_transformers'''])
     self.assertIn('''FlaxStableDiffusionPipeline'''							,  objects['''flax_and_transformers'''])
     self.assertIn('''LMSDiscreteScheduler'''							,  objects['''torch_and_scipy'''])
     self.assertIn('''OnnxStableDiffusionPipeline'''							,  objects['''torch_and_transformers_and_onnx'''])


 def  snake_case__				(	self) ->		List[str]:

     """simple docstring"""



     _UpperCAmelCase				:      Tuple			  =				create_dummy_object('''CONSTANT'''							,  '''\'torch\'''')
     self.assertEqual(_A							,  '''\nCONSTANT = None\n''')

     _UpperCAmelCase				:      Optional[int]			  =				create_dummy_object('''function'''							,  '''\'torch\'''')
     self.assertEqual(
         _A							,  '''\ndef function(*args, **kwargs):\n    requires_backends(function, \'torch\')\n''')

     _UpperCAmelCase				:      str			  =				'''
class FakeClass(metaclass=DummyObject):
    _backends = \'torch\'

    def __init__(self, *args, **kwargs):
        requires_backends(self, \'torch\')

    @classmethod
    def from_config(cls, *args, **kwargs):
        requires_backends(cls, \'torch\')

    @classmethod
    def from_pretrained(cls, *args, **kwargs):
        requires_backends(cls, \'torch\')
'''
     _UpperCAmelCase				:      int			  =				create_dummy_object('''FakeClass'''							,  '''\'torch\'''')
     self.assertEqual(_A							,  _A)


 def  snake_case__				(	self) ->		List[str]:

     """simple docstring"""



     _UpperCAmelCase				:      Any			  =				'''# This file is autogenerated by the command `make fix-copies`, do not edit.
from ..utils import DummyObject, requires_backends


CONSTANT = None


def function(*args, **kwargs):
    requires_backends(function, ["torch"])


class FakeClass(metaclass=DummyObject):
    _backends = ["torch"]

    def __init__(self, *args, **kwargs):
        requires_backends(self, ["torch"])

    @classmethod
    def from_config(cls, *args, **kwargs):
        requires_backends(cls, ["torch"])

    @classmethod
    def from_pretrained(cls, *args, **kwargs):
        requires_backends(cls, ["torch"])
'''
     _UpperCAmelCase				:      Any			  =				create_dummy_files({'''torch''': ['''CONSTANT''', '''function''', '''FakeClass''']})
     self.assertEqual(dummy_files['''torch''']							,  _A)












import unittest

from datasets import load_dataset

from transformers import BloomTokenizerFast
from transformers.testing_utils import require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin
@require_tokenizers
class 							A_    (						__lowercase ,   unittest.TestCase			):






 '''simple docstring'''




 _SCREAMING_SNAKE_CASE							:			int =					None
 _SCREAMING_SNAKE_CASE							:			str =					BloomTokenizerFast
 _SCREAMING_SNAKE_CASE							:			str =					BloomTokenizerFast
 _SCREAMING_SNAKE_CASE							:			Any =					True
 _SCREAMING_SNAKE_CASE							:			Optional[int] =					False
 _SCREAMING_SNAKE_CASE							:			Optional[int] =					"tokenizer_file"
 _SCREAMING_SNAKE_CASE							:			Optional[Any] =					{"bos_token": "<s>", "eos_token": "</s>", "unk_token": "<unk>", "pad_token": "<pad>"}


 def  snake_case__				(	self) ->		Dict:

     """simple docstring"""



     super().setUp()
     _UpperCAmelCase				:      List[Any]			  =				BloomTokenizerFast.from_pretrained('''bigscience/tokenizer''')
     tokenizer.save_pretrained(self.tmpdirname)


 def  snake_case__				(	self							,  **_A) ->		List[Any]:

     """simple docstring"""



     kwargs.update(self.special_tokens_map)
     return BloomTokenizerFast.from_pretrained(self.tmpdirname							,  **_A)


 def  snake_case__				(	self) ->		List[str]:

     """simple docstring"""



     _UpperCAmelCase				:      Optional[Any]			  =				self.get_rust_tokenizer()

     _UpperCAmelCase				:      int			  =				['''The quick brown fox</s>''', '''jumps over the lazy dog</s>''']
     _UpperCAmelCase				:      List[Any]			  =				[[2175, 23714, 73173, 144252, 2], [77, 132619, 3478, 368, 109586, 35433, 2]]

     _UpperCAmelCase				:      Any			  =				tokenizer.batch_encode_plus(_A)['''input_ids''']
     self.assertListEqual(_A							,  _A)

     _UpperCAmelCase				:      Any			  =				tokenizer.batch_decode(_A)
     self.assertListEqual(_A							,  _A)


 def  snake_case__				(	self							,  _A=6) ->		Optional[Any]:

     """simple docstring"""



     for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
         with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''):
             _UpperCAmelCase				:      Optional[int]			  =				self.rust_tokenizer_class.from_pretrained(_A							,  **_A)
             # tokenizer_r.pad_token = None # Hotfixing padding = None
             # Simple input
             _UpperCAmelCase				:      List[Any]			  =				'''This is a simple input'''
             _UpperCAmelCase				:      Any			  =				['''This is a simple input 1''', '''This is a simple input 2''']
             _UpperCAmelCase				:      int			  =				('''This is a simple input''', '''This is a pair''')
             _UpperCAmelCase				:      Optional[int]			  =				[
                 ('''This is a simple input 1''', '''This is a simple input 2'''),
                 ('''This is a simple pair 1''', '''This is a simple pair 2'''),
             ]

             # Simple input tests
             try:
                 tokenizer_r.encode(_A							,  max_length=_A)
                 tokenizer_r.encode_plus(_A							,  max_length=_A)

                 tokenizer_r.batch_encode_plus(_A							,  max_length=_A)
                 tokenizer_r.encode(_A							,  max_length=_A)
                 tokenizer_r.batch_encode_plus(_A							,  max_length=_A)
             except ValueError:
                 self.fail('''Bloom Tokenizer should be able to deal with padding''')

             _UpperCAmelCase				:      Tuple			  =				None  # Hotfixing padding = None
             self.assertRaises(_A							,  tokenizer_r.encode							,  _A							,  max_length=_A							,  padding='''max_length''')

             # Simple input
             self.assertRaises(_A							,  tokenizer_r.encode_plus							,  _A							,  max_length=_A							,  padding='''max_length''')

             # Simple input
             self.assertRaises(
                 _A							,  tokenizer_r.batch_encode_plus							,  _A							,  max_length=_A							,  padding='''max_length'''							,  )

             # Pair input
             self.assertRaises(_A							,  tokenizer_r.encode							,  _A							,  max_length=_A							,  padding='''max_length''')

             # Pair input
             self.assertRaises(_A							,  tokenizer_r.encode_plus							,  _A							,  max_length=_A							,  padding='''max_length''')

             # Pair input
             self.assertRaises(
                 _A							,  tokenizer_r.batch_encode_plus							,  _A							,  max_length=_A							,  padding='''max_length'''							,  )


 def  snake_case__				(	self) ->		Any:

     """simple docstring"""



     _UpperCAmelCase				:      Dict			  =				self.get_rust_tokenizer()
     _UpperCAmelCase				:      int			  =				load_dataset('''xnli'''							,  '''all_languages'''							,  split='''test'''							,  streaming=_A)

     _UpperCAmelCase				:      Tuple			  =				next(iter(_A))['''premise''']  # pick up one data
     _UpperCAmelCase				:      List[Any]			  =				list(sample_data.values())

     _UpperCAmelCase				:      Any			  =				list(map(tokenizer.encode							,  _A))
     _UpperCAmelCase				:      List[str]			  =				[tokenizer.decode(_A							,  clean_up_tokenization_spaces=_A) for x in output_tokens]
     self.assertListEqual(_A							,  _A)


 def  snake_case__				(	self) ->		Optional[Any]:

     """simple docstring"""



     self.assertGreaterEqual(len(self.tokenizer_class.pretrained_vocab_files_map)							,  1)
     self.assertGreaterEqual(len(list(self.tokenizer_class.pretrained_vocab_files_map.values())[0])							,  1)




