






import gc
import unittest

from parameterized import parameterized

from diffusers import FlaxUNetaDConditionModel
from diffusers.utils import is_flax_available
from diffusers.utils.testing_utils import load_hf_numpy, require_flax, slow


if is_flax_available():
					import jax
					import jax.numpy as jnp
@slow
@require_flax
class      UpperCamelCase (				unittest.TestCase ):







		"""simple docstring"""



		def       A(  self		:       Union[str, Any]						,_SCREAMING_SNAKE_CASE		:       Dict						,_SCREAMING_SNAKE_CASE		:       Dict		)    ->   Optional[int]:


						'''simple docstring'''
						return f'gaussian_noise_s={seed}_shape={"_".join([str(_SCREAMING_SNAKE_CASE		) for s in shape]		)}.npy'
		def       A(  self		:       Tuple		)    ->   Tuple:


						'''simple docstring'''
						super().tearDown()
						gc.collect()
		def       A(  self		:       str						,_SCREAMING_SNAKE_CASE		:       Dict=0						,_SCREAMING_SNAKE_CASE		:       Optional[Any]=(4, 4, 6_4, 6_4)						,_SCREAMING_SNAKE_CASE		:       Union[str, Any]=False		)    ->   int:


						'''simple docstring'''
						A	 =      jnp.bfloataa if fpaa else jnp.floataa
						A	 =      jnp.array(load_hf_numpy(self.get_file_format(_SCREAMING_SNAKE_CASE						,_SCREAMING_SNAKE_CASE		)		)						,dtype=_SCREAMING_SNAKE_CASE		)
						return image
		def       A(  self		:       str						,_SCREAMING_SNAKE_CASE		:       int=False						,_SCREAMING_SNAKE_CASE		:       Tuple="CompVis/stable-diffusion-v1-4"		)    ->   Optional[int]:


						'''simple docstring'''
						A	 =      jnp.bfloataa if fpaa else jnp.floataa
						A	 =      'bf16' if fpaa else None

						A ,		A	 =      FlaxUNetaDConditionModel.from_pretrained(
						    _SCREAMING_SNAKE_CASE						,subfolder='unet'						,dtype=_SCREAMING_SNAKE_CASE						,revision=_SCREAMING_SNAKE_CASE		)
						return model, params
		def       A(  self		:       Optional[Any]						,_SCREAMING_SNAKE_CASE		:       Optional[Any]=0						,_SCREAMING_SNAKE_CASE		:       str=(4, 7_7, 7_6_8)						,_SCREAMING_SNAKE_CASE		:       Optional[Any]=False		)    ->   List[str]:


						'''simple docstring'''
						A	 =      jnp.bfloataa if fpaa else jnp.floataa
						A	 =      jnp.array(load_hf_numpy(self.get_file_format(_SCREAMING_SNAKE_CASE						,_SCREAMING_SNAKE_CASE		)		)						,dtype=_SCREAMING_SNAKE_CASE		)
						return hidden_states
		@parameterized.expand(
		    [
		        # fmt: off
		        [8_3, 4, [-0.2323, -0.1304, 0.0813, -0.3093, -0.0919, -0.1571, -0.1125, -0.5806]],
		        [1_7, 0.55, [-0.0831, -0.2443, 0.0901, -0.0919, 0.3396, 0.0103, -0.3743, 0.0701]],
		        [8, 0.89, [-0.4863, 0.0859, 0.0875, -0.1658, 0.9199, -0.0114, 0.4839, 0.4639]],
		        [3, 1_0_0_0, [-0.5649, 0.2402, -0.5518, 0.1248, 1.1328, -0.2443, -0.0325, -1.0078]],
		        # fmt: on
		    ]		)
		def       A(  self		:       Union[str, Any]						,_SCREAMING_SNAKE_CASE		:       Tuple						,_SCREAMING_SNAKE_CASE		:       Dict						,_SCREAMING_SNAKE_CASE		:       List[str]		)    ->   Tuple:


						'''simple docstring'''
						A ,		A	 =      self.get_unet_model(model_id='CompVis/stable-diffusion-v1-4'						,fpaa=_SCREAMING_SNAKE_CASE		)
						A	 =      self.get_latents(_SCREAMING_SNAKE_CASE						,fpaa=_SCREAMING_SNAKE_CASE		)
						A	 =      self.get_encoder_hidden_states(_SCREAMING_SNAKE_CASE						,fpaa=_SCREAMING_SNAKE_CASE		)

						A	 =      model.apply(
						    {'params': params}						,_SCREAMING_SNAKE_CASE						,jnp.array(_SCREAMING_SNAKE_CASE						,dtype=jnp.intaa		)						,encoder_hidden_states=_SCREAMING_SNAKE_CASE						,).sample

						assert sample.shape == latents.shape

						A	 =      jnp.asarray(jax.device_get((sample[-1, -2:, -2:, :2].flatten())		)						,dtype=jnp.floataa		)
						A	 =      jnp.array(_SCREAMING_SNAKE_CASE						,dtype=jnp.floataa		)

						# Found torch (float16) and flax (bfloat16) outputs to be within this tolerance, in the same hardware
						assert jnp.allclose(_SCREAMING_SNAKE_CASE						,_SCREAMING_SNAKE_CASE						,atol=1E-2		)







		@parameterized.expand(
		    [
		        # fmt: off
		        [8_3, 4, [0.1514, 0.0807, 0.1624, 0.1016, -0.1896, 0.0263, 0.0677, 0.2310]],
		        [1_7, 0.55, [0.1164, -0.0216, 0.0170, 0.1589, -0.3120, 0.1005, -0.0581, -0.1458]],
		        [8, 0.89, [-0.1758, -0.0169, 0.1004, -0.1411, 0.1312, 0.1103, -0.1996, 0.2139]],
		        [3, 1_0_0_0, [0.1214, 0.0352, -0.0731, -0.1562, -0.0994, -0.0906, -0.2340, -0.0539]],
		        # fmt: on
		    ]		)
		def       A(  self		:       Any						,_SCREAMING_SNAKE_CASE		:       List[str]						,_SCREAMING_SNAKE_CASE		:       Dict						,_SCREAMING_SNAKE_CASE		:       Tuple		)    ->   Tuple:


						'''simple docstring'''
						A ,		A	 =      self.get_unet_model(model_id='stabilityai/stable-diffusion-2'						,fpaa=_SCREAMING_SNAKE_CASE		)
						A	 =      self.get_latents(_SCREAMING_SNAKE_CASE						,shape=(4, 4, 9_6, 9_6)						,fpaa=_SCREAMING_SNAKE_CASE		)
						A	 =      self.get_encoder_hidden_states(_SCREAMING_SNAKE_CASE						,shape=(4, 7_7, 1_0_2_4)						,fpaa=_SCREAMING_SNAKE_CASE		)

						A	 =      model.apply(
						    {'params': params}						,_SCREAMING_SNAKE_CASE						,jnp.array(_SCREAMING_SNAKE_CASE						,dtype=jnp.intaa		)						,encoder_hidden_states=_SCREAMING_SNAKE_CASE						,).sample

						assert sample.shape == latents.shape

						A	 =      jnp.asarray(jax.device_get((sample[-1, -2:, -2:, :2].flatten())		)						,dtype=jnp.floataa		)
						A	 =      jnp.array(_SCREAMING_SNAKE_CASE						,dtype=jnp.floataa		)

						# Found torch (float16) and flax (bfloat16) outputs to be within this tolerance, on the same hardware
						assert jnp.allclose(_SCREAMING_SNAKE_CASE						,_SCREAMING_SNAKE_CASE						,atol=1E-2		)













import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils import AddedToken
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
					from .tokenization_albert import AlbertTokenizer
else:
					lowerCAmelCase_           = None

lowerCAmelCase_           = logging.get_logger(__name__)
lowerCAmelCase_           = {'vocab_file': 'spiece.model', 'tokenizer_file': 'tokenizer.json'}

lowerCAmelCase_           = {
    'vocab_file': {
        'albert-base-v1': 'https://huggingface.co/albert-base-v1/resolve/main/spiece.model',
        'albert-large-v1': 'https://huggingface.co/albert-large-v1/resolve/main/spiece.model',
        'albert-xlarge-v1': 'https://huggingface.co/albert-xlarge-v1/resolve/main/spiece.model',
        'albert-xxlarge-v1': 'https://huggingface.co/albert-xxlarge-v1/resolve/main/spiece.model',
        'albert-base-v2': 'https://huggingface.co/albert-base-v2/resolve/main/spiece.model',
        'albert-large-v2': 'https://huggingface.co/albert-large-v2/resolve/main/spiece.model',
        'albert-xlarge-v2': 'https://huggingface.co/albert-xlarge-v2/resolve/main/spiece.model',
        'albert-xxlarge-v2': 'https://huggingface.co/albert-xxlarge-v2/resolve/main/spiece.model',
    },
    'tokenizer_file': {
        'albert-base-v1': 'https://huggingface.co/albert-base-v1/resolve/main/tokenizer.json',
        'albert-large-v1': 'https://huggingface.co/albert-large-v1/resolve/main/tokenizer.json',
        'albert-xlarge-v1': 'https://huggingface.co/albert-xlarge-v1/resolve/main/tokenizer.json',
        'albert-xxlarge-v1': 'https://huggingface.co/albert-xxlarge-v1/resolve/main/tokenizer.json',
        'albert-base-v2': 'https://huggingface.co/albert-base-v2/resolve/main/tokenizer.json',
        'albert-large-v2': 'https://huggingface.co/albert-large-v2/resolve/main/tokenizer.json',
        'albert-xlarge-v2': 'https://huggingface.co/albert-xlarge-v2/resolve/main/tokenizer.json',
        'albert-xxlarge-v2': 'https://huggingface.co/albert-xxlarge-v2/resolve/main/tokenizer.json',
    },
}

lowerCAmelCase_           = {
    'albert-base-v1': 512,
    'albert-large-v1': 512,
    'albert-xlarge-v1': 512,
    'albert-xxlarge-v1': 512,
    'albert-base-v2': 512,
    'albert-large-v2': 512,
    'albert-xlarge-v2': 512,
    'albert-xxlarge-v2': 512,
}

lowerCAmelCase_           = '▁'
class      UpperCamelCase (				snake_case__ ):







		"""simple docstring"""



		snake_case    =   VOCAB_FILES_NAMES
		snake_case    =   PRETRAINED_VOCAB_FILES_MAP
		snake_case    =   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
		snake_case    =   AlbertTokenizer
		def __init__(  self		:       str						,_SCREAMING_SNAKE_CASE		:       int=None						,_SCREAMING_SNAKE_CASE		:       List[str]=None						,_SCREAMING_SNAKE_CASE		:       List[str]=True						,_SCREAMING_SNAKE_CASE		:       str=True						,_SCREAMING_SNAKE_CASE		:       Optional[int]=False						,_SCREAMING_SNAKE_CASE		:       List[str]="[CLS]"						,_SCREAMING_SNAKE_CASE		:       Optional[int]="[SEP]"						,_SCREAMING_SNAKE_CASE		:       Tuple="<unk>"						,_SCREAMING_SNAKE_CASE		:       Union[str, Any]="[SEP]"						,_SCREAMING_SNAKE_CASE		:       Tuple="<pad>"						,_SCREAMING_SNAKE_CASE		:       Any="[CLS]"						,_SCREAMING_SNAKE_CASE		:       Optional[int]="[MASK]"						,**_SCREAMING_SNAKE_CASE		:       Any						,)    ->   Tuple:


						'''simple docstring'''
						# Mask token behave like a normal word, i.e. include the space before it and
						# is included in the raw text, there should be a match in a non-normalized sentence.
						A	 =      (
						    AddedToken(_SCREAMING_SNAKE_CASE						,lstrip=_SCREAMING_SNAKE_CASE						,rstrip=_SCREAMING_SNAKE_CASE						,normalized=_SCREAMING_SNAKE_CASE		)
						    if isinstance(_SCREAMING_SNAKE_CASE						,_SCREAMING_SNAKE_CASE		)
						    else mask_token
						)

						super().__init__(
						    _SCREAMING_SNAKE_CASE						,tokenizer_file=_SCREAMING_SNAKE_CASE						,do_lower_case=_SCREAMING_SNAKE_CASE						,remove_space=_SCREAMING_SNAKE_CASE						,keep_accents=_SCREAMING_SNAKE_CASE						,bos_token=_SCREAMING_SNAKE_CASE						,eos_token=_SCREAMING_SNAKE_CASE						,unk_token=_SCREAMING_SNAKE_CASE						,sep_token=_SCREAMING_SNAKE_CASE						,pad_token=_SCREAMING_SNAKE_CASE						,cls_token=_SCREAMING_SNAKE_CASE						,mask_token=_SCREAMING_SNAKE_CASE						,**_SCREAMING_SNAKE_CASE						,)

						A	 =      do_lower_case
						A	 =      remove_space
						A	 =      keep_accents
						A	 =      vocab_file
						A	 =      False if not self.vocab_file else True
		def       A(  self		:       int						,_SCREAMING_SNAKE_CASE		:       List[int]						,_SCREAMING_SNAKE_CASE		:       Optional[List[int]] = None		)    ->   List[int]:


						'''simple docstring'''
						A	 =      [self.sep_token_id]
						A	 =      [self.cls_token_id]
						if token_ids_a is None:
										return cls + token_ids_a + sep
						return cls + token_ids_a + sep + token_ids_a + sep
		def       A(  self		:       List[Any]						,_SCREAMING_SNAKE_CASE		:       List[int]						,_SCREAMING_SNAKE_CASE		:       Optional[List[int]] = None		)    ->   List[int]:


						'''simple docstring'''
						A	 =      [self.sep_token_id]
						A	 =      [self.cls_token_id]

						if token_ids_a is None:
										return len(cls + token_ids_a + sep		) * [0]
						return len(cls + token_ids_a + sep		) * [0] + len(token_ids_a + sep		) * [1]







		def       A(  self		:       Union[str, Any]						,_SCREAMING_SNAKE_CASE		:       str						,_SCREAMING_SNAKE_CASE		:       Optional[str] = None		)    ->   Tuple[str]:


						'''simple docstring'''
						if not self.can_save_slow_tokenizer:
										raise ValueError(
										    'Your fast tokenizer does not have the necessary information to save the vocabulary for a slow '
										    'tokenizer.'		)

						if not os.path.isdir(_SCREAMING_SNAKE_CASE		):
										logger.error(f'Vocabulary path ({save_directory}) should be a directory'		)
										return
						A	 =      os.path.join(
						    _SCREAMING_SNAKE_CASE						,(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']		)

						if os.path.abspath(self.vocab_file		) != os.path.abspath(_SCREAMING_SNAKE_CASE		):
										copyfile(self.vocab_file						,_SCREAMING_SNAKE_CASE		)

						return (out_vocab_file,)





