


"""simple docstring"""






import unittest

from diffusers.pipelines.pipeline_utils import is_safetensors_compatible


class     __lowerCamelCase  (					unittest.TestCase				):

     '''simple docstring'''


     def 							_UpperCAmelCase      (      self							)       ->							Tuple:
          _a    	=				[
              '''safety_checker/pytorch_model.bin''',
              '''safety_checker/model.safetensors''',
              '''vae/diffusion_pytorch_model.bin''',
              '''vae/diffusion_pytorch_model.safetensors''',
              '''text_encoder/pytorch_model.bin''',
              '''text_encoder/model.safetensors''',
              '''unet/diffusion_pytorch_model.bin''',
              '''unet/diffusion_pytorch_model.safetensors''',
          ]
          self.assertTrue(is_safetensors_compatible(__UpperCAmelCase							)							)


     def 							_UpperCAmelCase      (      self							)       ->							List[str]:
          _a    	=				[
              '''unet/diffusion_pytorch_model.bin''',
              '''unet/diffusion_pytorch_model.safetensors''',
          ]
          self.assertTrue(is_safetensors_compatible(__UpperCAmelCase							)							)


     def 							_UpperCAmelCase      (      self							)       ->							int:
          _a    	=				[
              '''safety_checker/pytorch_model.bin''',
              '''safety_checker/model.safetensors''',
              '''vae/diffusion_pytorch_model.bin''',
              '''vae/diffusion_pytorch_model.safetensors''',
              '''text_encoder/pytorch_model.bin''',
              '''text_encoder/model.safetensors''',
              '''unet/diffusion_pytorch_model.bin''',
              # Removed: 'unet/diffusion_pytorch_model.safetensors',
          ]
          self.assertFalse(is_safetensors_compatible(__UpperCAmelCase							)							)


     def 							_UpperCAmelCase      (      self							)       ->							List[Any]:
          _a    	=				[
              '''text_encoder/pytorch_model.bin''',
              '''text_encoder/model.safetensors''',
          ]
          self.assertTrue(is_safetensors_compatible(__UpperCAmelCase							)							)


     def 							_UpperCAmelCase      (      self							)       ->							str:
          _a    	=				[
              '''safety_checker/pytorch_model.bin''',
              '''safety_checker/model.safetensors''',
              '''vae/diffusion_pytorch_model.bin''',
              '''vae/diffusion_pytorch_model.safetensors''',
              '''text_encoder/pytorch_model.bin''',
              # Removed: 'text_encoder/model.safetensors',
              '''unet/diffusion_pytorch_model.bin''',
              '''unet/diffusion_pytorch_model.safetensors''',
          ]
          self.assertFalse(is_safetensors_compatible(__UpperCAmelCase							)							)


     def 							_UpperCAmelCase      (      self							)       ->							List[Any]:
          _a    	=				[
              '''safety_checker/pytorch_model.fp16.bin''',
              '''safety_checker/model.fp16.safetensors''',
              '''vae/diffusion_pytorch_model.fp16.bin''',
              '''vae/diffusion_pytorch_model.fp16.safetensors''',
              '''text_encoder/pytorch_model.fp16.bin''',
              '''text_encoder/model.fp16.safetensors''',
              '''unet/diffusion_pytorch_model.fp16.bin''',
              '''unet/diffusion_pytorch_model.fp16.safetensors''',
          ]
          _a    	=				'''fp16'''
          self.assertTrue(is_safetensors_compatible(__UpperCAmelCase    ,				variant=__UpperCAmelCase							)							)


     def 							_UpperCAmelCase      (      self							)       ->							Optional[Any]:
          _a    	=				[
              '''unet/diffusion_pytorch_model.fp16.bin''',
              '''unet/diffusion_pytorch_model.fp16.safetensors''',
          ]
          _a    	=				'''fp16'''
          self.assertTrue(is_safetensors_compatible(__UpperCAmelCase    ,				variant=__UpperCAmelCase							)							)


     def 							_UpperCAmelCase      (      self							)       ->							str:
          # pass variant but use the non-variant filenames
          _a    	=				[
              '''unet/diffusion_pytorch_model.bin''',
              '''unet/diffusion_pytorch_model.safetensors''',
          ]
          _a    	=				'''fp16'''
          self.assertTrue(is_safetensors_compatible(__UpperCAmelCase    ,				variant=__UpperCAmelCase							)							)


     def 							_UpperCAmelCase      (      self							)       ->							Optional[int]:
          _a    	=				[
              '''safety_checker/pytorch_model.fp16.bin''',
              '''safety_checker/model.fp16.safetensors''',
              '''vae/diffusion_pytorch_model.fp16.bin''',
              '''vae/diffusion_pytorch_model.fp16.safetensors''',
              '''text_encoder/pytorch_model.fp16.bin''',
              '''text_encoder/model.fp16.safetensors''',
              '''unet/diffusion_pytorch_model.fp16.bin''',
              # Removed: 'unet/diffusion_pytorch_model.fp16.safetensors',
          ]
          _a    	=				'''fp16'''
          self.assertFalse(is_safetensors_compatible(__UpperCAmelCase    ,				variant=__UpperCAmelCase							)							)


     def 							_UpperCAmelCase      (      self							)       ->							int:
          _a    	=				[
              '''text_encoder/pytorch_model.fp16.bin''',
              '''text_encoder/model.fp16.safetensors''',
          ]
          _a    	=				'''fp16'''
          self.assertTrue(is_safetensors_compatible(__UpperCAmelCase    ,				variant=__UpperCAmelCase							)							)


     def 							_UpperCAmelCase      (      self							)       ->							Union[str, Any]:
          # pass variant but use the non-variant filenames
          _a    	=				[
              '''text_encoder/pytorch_model.bin''',
              '''text_encoder/model.safetensors''',
          ]
          _a    	=				'''fp16'''
          self.assertTrue(is_safetensors_compatible(__UpperCAmelCase    ,				variant=__UpperCAmelCase							)							)





     def 							_UpperCAmelCase      (      self							)       ->							Union[str, Any]:
          _a    	=				[
              '''safety_checker/pytorch_model.fp16.bin''',
              '''safety_checker/model.fp16.safetensors''',
              '''vae/diffusion_pytorch_model.fp16.bin''',
              '''vae/diffusion_pytorch_model.fp16.safetensors''',
              '''text_encoder/pytorch_model.fp16.bin''',
              # 'text_encoder/model.fp16.safetensors',
              '''unet/diffusion_pytorch_model.fp16.bin''',
              '''unet/diffusion_pytorch_model.fp16.safetensors''',
          ]
          _a    	=				'''fp16'''
          self.assertFalse(is_safetensors_compatible(__UpperCAmelCase    ,				variant=__UpperCAmelCase							)							)



"""simple docstring"""






import warnings
from typing import Dict, List, Optional, Tuple

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


__snake_case		=       logging.get_logger(__name__)


class     __lowerCamelCase  (					a__				):

     '''simple docstring'''
     A_       :	Any    =		['input_ids', 'attention_mask']


     def __init__(      self    ,				__UpperCAmelCase="</s>"    ,				__UpperCAmelCase="<unk>"    ,				__UpperCAmelCase="<pad>"    ,				__UpperCAmelCase=125    ,				__UpperCAmelCase=None    ,				**__UpperCAmelCase    ,				)       ->							None:
          # Add extra_ids to the special token list
          if extra_ids > 0 and additional_special_tokens is None:
               _a    	=				[F'<extra_id_{i}>' for i in range(__UpperCAmelCase							)]
          elif extra_ids > 0 and additional_special_tokens is not None:
               # Check that we have the right number of extra_id special tokens
               _a    	=				len(set(filter(lambda __UpperCAmelCase							: bool('''extra_id''' in str(__UpperCAmelCase							)							)    ,				__UpperCAmelCase							)							)							)
               if extra_tokens != extra_ids:
                    raise ValueError(
                        F'Both extra_ids ({extra_ids}) and additional_special_tokens ({additional_special_tokens}) are'
                        ''' provided to ByT5Tokenizer. In this case the additional_special_tokens must include the'''
                        ''' extra_ids tokens'''							)

          _a    	=				AddedToken(__UpperCAmelCase    ,				lstrip=__UpperCAmelCase    ,				rstrip=__UpperCAmelCase							) if isinstance(__UpperCAmelCase    ,				__UpperCAmelCase							) else pad_token
          _a    	=				AddedToken(__UpperCAmelCase    ,				lstrip=__UpperCAmelCase    ,				rstrip=__UpperCAmelCase							) if isinstance(__UpperCAmelCase    ,				__UpperCAmelCase							) else eos_token
          _a    	=				AddedToken(__UpperCAmelCase    ,				lstrip=__UpperCAmelCase    ,				rstrip=__UpperCAmelCase							) if isinstance(__UpperCAmelCase    ,				__UpperCAmelCase							) else unk_token

          super().__init__(
              eos_token=__UpperCAmelCase    ,				unk_token=__UpperCAmelCase    ,				pad_token=__UpperCAmelCase    ,				extra_ids=__UpperCAmelCase    ,				additional_special_tokens=__UpperCAmelCase    ,				**__UpperCAmelCase    ,				)

          _a    	=				extra_ids

          _a    	=				2**8  # utf is 8 bits

          # define special tokens dict
          _a    	=				{
              self.pad_token: 0,
              self.eos_token: 1,
              self.unk_token: 2,
          }
          _a    	=				len(self.special_tokens_encoder							)
          _a    	=				len(__UpperCAmelCase							)
          for i, token in enumerate(__UpperCAmelCase							):
               _a    	=				self.vocab_size + i - n
          _a    	=				{v: k for k, v in self.special_tokens_encoder.items()}


     @property
     def 							_UpperCAmelCase      (      self							)       ->							Dict:
          return self._utf_vocab_size + self._num_special_tokens + self._extra_ids


     def 							_UpperCAmelCase      (      self    ,				__UpperCAmelCase    ,				__UpperCAmelCase = None    ,				__UpperCAmelCase = False							)       ->							List[int]:
          if already_has_special_tokens:
               return super().get_special_tokens_mask(
                   token_ids_a=__UpperCAmelCase    ,				token_ids_a=__UpperCAmelCase    ,				already_has_special_tokens=__UpperCAmelCase							)

          # normal case: some special tokens
          if token_ids_a is None:
               return ([0] * len(__UpperCAmelCase							)) + [1]
          return ([0] * len(__UpperCAmelCase							)) + [1] + ([0] * len(__UpperCAmelCase							)) + [1]


     def 							_UpperCAmelCase      (      self    ,				__UpperCAmelCase							)       ->							List[int]:
          if len(__UpperCAmelCase							) > 0 and token_ids[-1] == self.eos_token_id:
               warnings.warn(
                   F'This sequence already has {self.eos_token}. In future versions this behavior may lead to duplicated'
                   ''' eos tokens being added.'''							)
               return token_ids
          else:
               return token_ids + [self.eos_token_id]


     def 							_UpperCAmelCase      (      self    ,				__UpperCAmelCase    ,				__UpperCAmelCase = None							)       ->							List[int]:
          _a    	=				[self.eos_token_id]

          if token_ids_a is None:
               return len(token_ids_a + eos							) * [0]
          return len(token_ids_a + eos + token_ids_a + eos							) * [0]


     def 							_UpperCAmelCase      (      self    ,				__UpperCAmelCase    ,				__UpperCAmelCase = None							)       ->							List[int]:
          _a    	=				self._add_eos_if_not_present(__UpperCAmelCase							)
          if token_ids_a is None:
               return token_ids_a
          else:
               _a    	=				self._add_eos_if_not_present(__UpperCAmelCase							)
               return token_ids_a + token_ids_a


     def 							_UpperCAmelCase      (      self    ,				__UpperCAmelCase							)       ->							List[str]:
          _a    	=				[chr(__UpperCAmelCase							) for i in text.encode('''utf-8'''							)]
          return tokens


     def 							_UpperCAmelCase      (      self    ,				__UpperCAmelCase							)       ->							Union[str, Any]:
          if token in self.special_tokens_encoder:
               _a    	=				self.special_tokens_encoder[token]
          elif token in self.added_tokens_encoder:
               _a    	=				self.added_tokens_encoder[token]
          elif len(__UpperCAmelCase							) != 1:
               _a    	=				self.unk_token_id
          else:
               _a    	=				ord(__UpperCAmelCase							) + self._num_special_tokens
          return token_id


     def 							_UpperCAmelCase      (      self    ,				__UpperCAmelCase							)       ->							List[Any]:
          if index in self.special_tokens_decoder:
               _a    	=				self.special_tokens_decoder[index]
          else:
               _a    	=				chr(index - self._num_special_tokens							)
          return token


     def 							_UpperCAmelCase      (      self    ,				__UpperCAmelCase							)       ->							Any:
          _a    	=				B''''''
          for token in tokens:
               if token in self.special_tokens_decoder:
                    _a    	=				self.special_tokens_decoder[token].encode('''utf-8'''							)
               elif token in self.added_tokens_decoder:
                    _a    	=				self.special_tokens_decoder[token].encode('''utf-8'''							)
               elif token in self.special_tokens_encoder:
                    _a    	=				token.encode('''utf-8'''							)
               elif token in self.added_tokens_encoder:
                    _a    	=				token.encode('''utf-8'''							)
               else:
                    _a    	=				bytes([ord(__UpperCAmelCase							)]							)
               bstring += tok_string
          _a    	=				bstring.decode('''utf-8'''    ,				errors='''ignore'''							)
          return string





     def 							_UpperCAmelCase      (      self    ,				__UpperCAmelCase    ,				__UpperCAmelCase = None							)       ->							Tuple[str]:
          return ()