






import random
import unittest

import numpy as np

import transformers
from transformers import is_flax_available, is_torch_available
from transformers.testing_utils import is_pt_flax_cross_test, require_flax


if is_flax_available():
     import os

     import jax.numpy as jnp
     from jax import jit

     from transformers import AutoTokenizer, FlaxAutoModelForCausalLM
     from transformers.modeling_flax_pytorch_utils import load_flax_weights_in_pytorch_model

     lowerCamelCase__							=     """0.12"""  # assumed parallelism: 8


if is_torch_available():
     import torch






def   UpperCamelCase  (       snake_case__     :   str	,snake_case__     :   Optional[int]	,snake_case__     :   int=None  ):




     '''simple docstring'''






     if rng is None:
          __snake_case			:int          =						random.Random()

     __snake_case			:Any          =						1
     for dim in shape:
          total_dims *= dim

     __snake_case			:List[Any]          =						[]
     for _ in range(snake_case__  ):
          values.append(rng.randint(0	,vocab_size - 1  )  )

     __snake_case			:List[str]          =						np.array(snake_case__	,dtype=jnp.intaa  ).reshape(snake_case__  )

     return output






def   UpperCamelCase  (       snake_case__     :   List[str]	,snake_case__     :   Dict=None  ):




     '''simple docstring'''






     __snake_case			:Optional[Any]          =						ids_tensor(snake_case__	,vocab_size=2	,rng=snake_case__  )
     # make sure that at least one token is attended to for each batch
     __snake_case			:int          =						1
     return attn_mask




@require_flax
class 			snake_case__   :

       '''simple docstring'''
       lowerCamelCase							:				str 				= None
       lowerCamelCase							:				Tuple 				= ()



       def 					__lowercase						(							self    ) ->      int:






            '''simple docstring'''





            __snake_case       , __snake_case			:List[Any]          =						self.model_tester.prepare_config_and_inputs_for_common()

            # cut to half length & take max batch_size 3
            __snake_case			:List[str]          =						2
            __snake_case			:Optional[Any]          =						inputs["""input_ids"""].shape[-1] // 2
            __snake_case			:Dict          =						inputs["""input_ids"""][:max_batch_size, :sequence_length]

            __snake_case			:str          =						jnp.ones_like(a__    )
            __snake_case			:str          =						attention_mask[:max_batch_size, :sequence_length]

            # generate max 5 tokens
            __snake_case			:Dict          =						input_ids.shape[-1] + 5
            if config.eos_token_id is not None and config.pad_token_id is None:
                 # hack to allow generate for models such as GPT2 as is done in `generate()`
                 __snake_case			:Optional[int]          =						config.eos_token_id
            return config, input_ids, attention_mask, max_length



       @is_pt_flax_cross_test
       def 					__lowercase						(							self    ) ->      Any:






            '''simple docstring'''





            __snake_case       , __snake_case       , __snake_case       , __snake_case			:Tuple          =						self._get_input_ids_and_config()
            __snake_case			:Dict          =						False
            __snake_case			:Tuple          =						max_length
            __snake_case			:Optional[Any]          =						0

            for model_class in self.all_generative_model_classes:
                 __snake_case			:str          =						model_class(a__    )

                 __snake_case			:Any          =						model_class.__name__[4:]  # Skip the "Flax" at the beginning
                 __snake_case			:Optional[Any]          =						getattr(a__			,						a__    )
                 __snake_case			:Any          =						pt_model_class(a__    ).eval()
                 __snake_case			:Optional[Any]          =						load_flax_weights_in_pytorch_model(a__			,						flax_model.params    )

                 __snake_case			:Union[str, Any]          =						flax_model.generate(a__    ).sequences
                 __snake_case			:Any          =						pt_model.generate(torch.tensor(a__			,						dtype=torch.long    )    )

                 if flax_generation_outputs.shape[-1] > pt_generation_outputs.shape[-1]:
                      __snake_case			:List[str]          =						flax_generation_outputs[:, : pt_generation_outputs.shape[-1]]

                 self.assertListEqual(pt_generation_outputs.numpy().tolist()			,						flax_generation_outputs.tolist()    )



       def 					__lowercase						(							self    ) ->      List[Any]:






            '''simple docstring'''





            __snake_case       , __snake_case       , __snake_case       , __snake_case			:int          =						self._get_input_ids_and_config()
            __snake_case			:Dict          =						False
            __snake_case			:Tuple          =						max_length

            for model_class in self.all_generative_model_classes:
                 __snake_case			:Any          =						model_class(a__    )

                 __snake_case			:Dict          =						model.generate(a__    ).sequences
                 self.assertEqual(generation_outputs.shape[-1]			,						a__    )

                 __snake_case			:List[str]          =						jit(model.generate    )
                 __snake_case			:List[Any]          =						jit_generate(a__    ).sequences

                 self.assertListEqual(generation_outputs.tolist()			,						jit_generation_outputs.tolist()    )



       def 					__lowercase						(							self    ) ->      Optional[int]:






            '''simple docstring'''





            __snake_case       , __snake_case       , __snake_case       , __snake_case			:Tuple          =						self._get_input_ids_and_config()
            __snake_case			:Any          =						True
            __snake_case			:Dict          =						max_length

            for model_class in self.all_generative_model_classes:
                 __snake_case			:List[str]          =						model_class(a__    )

                 __snake_case			:Any          =						model.generate(a__    ).sequences
                 self.assertEqual(generation_outputs.shape[-1]			,						a__    )

                 __snake_case			:int          =						jit(model.generate    )
                 __snake_case			:Tuple          =						jit_generate(a__    ).sequences

                 self.assertListEqual(generation_outputs.tolist()			,						jit_generation_outputs.tolist()    )



       def 					__lowercase						(							self    ) ->      Optional[Any]:






            '''simple docstring'''





            __snake_case       , __snake_case       , __snake_case       , __snake_case			:Tuple          =						self._get_input_ids_and_config()
            __snake_case			:Optional[int]          =						False
            __snake_case			:Optional[Any]          =						max_length
            __snake_case			:Tuple          =						2

            for model_class in self.all_generative_model_classes:
                 __snake_case			:List[Any]          =						model_class(a__    )

                 __snake_case			:Union[str, Any]          =						model.generate(a__    ).sequences
                 self.assertEqual(generation_outputs.shape[-1]			,						a__    )

                 __snake_case			:List[str]          =						jit(model.generate    )
                 __snake_case			:str          =						jit_generate(a__    ).sequences

                 self.assertListEqual(generation_outputs.tolist()			,						jit_generation_outputs.tolist()    )



       def 					__lowercase						(							self    ) ->      Dict:






            '''simple docstring'''





            __snake_case       , __snake_case       , __snake_case       , __snake_case			:List[str]          =						self._get_input_ids_and_config()
            __snake_case			:int          =						False
            __snake_case			:int          =						max_length
            __snake_case			:str          =						2
            __snake_case			:Optional[Any]          =						2

            for model_class in self.all_generative_model_classes:
                 __snake_case			:Union[str, Any]          =						model_class(a__    )

                 __snake_case			:Optional[Any]          =						model.generate(a__    ).sequences
                 self.assertEqual(generation_outputs.shape[0]			,						input_ids.shape[0] * config.num_return_sequences    )



       def 					__lowercase						(							self    ) ->      Dict:






            '''simple docstring'''





            __snake_case       , __snake_case       , __snake_case       , __snake_case			:Tuple          =						self._get_input_ids_and_config()
            __snake_case			:List[Any]          =						True
            __snake_case			:int          =						max_length
            __snake_case			:List[str]          =						0.8
            __snake_case			:str          =						10
            __snake_case			:Tuple          =						0.3
            __snake_case			:str          =						1
            __snake_case			:int          =						8
            __snake_case			:Union[str, Any]          =						9

            for model_class in self.all_generative_model_classes:
                 __snake_case			:str          =						model_class(a__    )

                 __snake_case			:Any          =						model.generate(a__    ).sequences
                 self.assertEqual(generation_outputs.shape[-1]			,						a__    )

                 __snake_case			:List[Any]          =						jit(model.generate    )
                 __snake_case			:Optional[int]          =						jit_generate(a__    ).sequences

                 self.assertListEqual(generation_outputs.tolist()			,						jit_generation_outputs.tolist()    )



       def 					__lowercase						(							self    ) ->      List[Any]:






            '''simple docstring'''





            __snake_case       , __snake_case       , __snake_case       , __snake_case			:Optional[int]          =						self._get_input_ids_and_config()
            __snake_case			:Optional[int]          =						max_length
            __snake_case			:List[str]          =						1
            __snake_case			:Tuple          =						8
            __snake_case			:int          =						9

            for model_class in self.all_generative_model_classes:
                 __snake_case			:Tuple          =						model_class(a__    )

                 __snake_case			:str          =						model.generate(a__    ).sequences
                 self.assertEqual(generation_outputs.shape[-1]			,						a__    )

                 __snake_case			:List[Any]          =						jit(model.generate    )
                 __snake_case			:List[Any]          =						jit_generate(a__    ).sequences

                 self.assertListEqual(generation_outputs.tolist()			,						jit_generation_outputs.tolist()    )



       def 					__lowercase						(							self    ) ->      Optional[Any]:






            '''simple docstring'''





            __snake_case       , __snake_case       , __snake_case       , __snake_case			:Optional[int]          =						self._get_input_ids_and_config()
            __snake_case			:Dict          =						max_length
            __snake_case			:int          =						2
            __snake_case			:List[Any]          =						1
            __snake_case			:List[Any]          =						8
            __snake_case			:Any          =						9

            for model_class in self.all_generative_model_classes:
                 __snake_case			:str          =						model_class(a__    )

                 __snake_case			:List[Any]          =						model.generate(a__    ).sequences
                 self.assertEqual(generation_outputs.shape[-1]			,						a__    )

                 __snake_case			:int          =						jit(model.generate    )
                 __snake_case			:Dict          =						jit_generate(a__    ).sequences

                 self.assertListEqual(generation_outputs.tolist()			,						jit_generation_outputs.tolist()    )



       def 					__lowercase						(							self    ) ->      Dict:






            '''simple docstring'''





            __snake_case       , __snake_case       , __snake_case       , __snake_case			:Any          =						self._get_input_ids_and_config()

            # pad attention mask on the left
            __snake_case			:List[Any]          =						attention_mask.at[(0, 0)].set(0    )

            __snake_case			:Union[str, Any]          =						False
            __snake_case			:List[Any]          =						max_length

            for model_class in self.all_generative_model_classes:
                 __snake_case			:Optional[Any]          =						model_class(a__    )

                 __snake_case			:List[Any]          =						model.generate(a__			,						attention_mask=a__    ).sequences
                 self.assertEqual(generation_outputs.shape[-1]			,						a__    )

                 __snake_case			:Tuple          =						jit(model.generate    )
                 __snake_case			:int          =						jit_generate(a__			,						attention_mask=a__    ).sequences

                 self.assertListEqual(generation_outputs.tolist()			,						jit_generation_outputs.tolist()    )



       def 					__lowercase						(							self    ) ->      Any:






            '''simple docstring'''





            __snake_case       , __snake_case       , __snake_case       , __snake_case			:int          =						self._get_input_ids_and_config()

            # pad attention mask on the left
            __snake_case			:Dict          =						attention_mask.at[(0, 0)].set(0    )

            __snake_case			:List[str]          =						True
            __snake_case			:Union[str, Any]          =						max_length

            for model_class in self.all_generative_model_classes:
                 __snake_case			:List[str]          =						model_class(a__    )

                 __snake_case			:Union[str, Any]          =						model.generate(a__			,						attention_mask=a__    ).sequences
                 self.assertEqual(generation_outputs.shape[-1]			,						a__    )

                 __snake_case			:Union[str, Any]          =						jit(model.generate    )
                 __snake_case			:Dict          =						jit_generate(a__			,						attention_mask=a__    ).sequences

                 self.assertListEqual(generation_outputs.tolist()			,						jit_generation_outputs.tolist()    )



       def 					__lowercase						(							self    ) ->      Dict:






            '''simple docstring'''





            __snake_case       , __snake_case       , __snake_case       , __snake_case			:Optional[int]          =						self._get_input_ids_and_config()

            # pad attention mask on the left
            __snake_case			:Union[str, Any]          =						attention_mask.at[(0, 0)].set(0    )

            __snake_case			:Any          =						2
            __snake_case			:Union[str, Any]          =						max_length

            for model_class in self.all_generative_model_classes:
                 __snake_case			:int          =						model_class(a__    )

                 __snake_case			:Any          =						model.generate(a__			,						attention_mask=a__    ).sequences
                 self.assertEqual(generation_outputs.shape[-1]			,						a__    )

                 __snake_case			:Optional[int]          =						jit(model.generate    )
                 __snake_case			:Dict          =						jit_generate(a__			,						attention_mask=a__    ).sequences

                 self.assertListEqual(generation_outputs.tolist()			,						jit_generation_outputs.tolist()    )




@require_flax
class 			snake_case__   (							unittest.TestCase):

       '''simple docstring'''



       def 					__lowercase						(							self    ) ->      Union[str, Any]:






            '''simple docstring'''





            __snake_case			:List[str]          =						AutoTokenizer.from_pretrained("""hf-internal-testing/tiny-bert"""    )
            __snake_case			:Union[str, Any]          =						FlaxAutoModelForCausalLM.from_pretrained("""hf-internal-testing/tiny-bert-flax-only"""    )

            __snake_case			:List[str]          =						"""Hello world"""
            __snake_case			:int          =						tokenizer(a__			,						return_tensors="""np"""    ).input_ids

            # typos are quickly detected (the correct argument is `do_sample`)
            with self.assertRaisesRegex(a__			,						"""do_samples"""    ):
                 model.generate(a__			,						do_samples=a__    )

            # arbitrary arguments that will not be used anywhere are also not accepted
            with self.assertRaisesRegex(a__			,						"""foo"""    ):
                 __snake_case			:Tuple          =						{"""foo""": """bar"""}
                 model.generate(a__			,						**a__    )










import copy
import os
from typing import TYPE_CHECKING, List, Union


if TYPE_CHECKING:
     pass

from ...configuration_utils import PretrainedConfig
from ...utils import logging


lowerCamelCase__							=     logging.get_logger(__name__)

lowerCamelCase__							=     {
    """kakaobrain/align-base""": """https://huggingface.co/kakaobrain/align-base/resolve/main/config.json""",
}




class 			snake_case__   (							lowercase_):

       '''simple docstring'''
       lowerCamelCase							:				Union[str, Any] 				= "align_text_model"



       def __init__(							self			,						a__=3_05_22			,						a__=7_68			,						a__=12			,						a__=12			,						a__=30_72			,						a__="gelu"			,						a__=0.1			,						a__=0.1			,						a__=5_12			,						a__=2			,						a__=0.02			,						a__=1e-12			,						a__=0			,						a__="absolute"			,						a__=True			,						**a__			,						) ->      List[str]:






            '''simple docstring'''





            super().__init__(**a__    )

            __snake_case			:Optional[int]          =						vocab_size
            __snake_case			:List[str]          =						hidden_size
            __snake_case			:Optional[Any]          =						num_hidden_layers
            __snake_case			:int          =						num_attention_heads
            __snake_case			:Optional[Any]          =						hidden_act
            __snake_case			:Union[str, Any]          =						intermediate_size
            __snake_case			:int          =						hidden_dropout_prob
            __snake_case			:Optional[Any]          =						attention_probs_dropout_prob
            __snake_case			:List[str]          =						max_position_embeddings
            __snake_case			:List[str]          =						type_vocab_size
            __snake_case			:Union[str, Any]          =						initializer_range
            __snake_case			:str          =						layer_norm_eps
            __snake_case			:Any          =						position_embedding_type
            __snake_case			:List[str]          =						use_cache
            __snake_case			:Optional[int]          =						pad_token_id



       @classmethod
       def 					__lowercase						(							cls			,						a__			,						**a__    ) ->      "PretrainedConfig":






            '''simple docstring'''





            cls._set_token_in_kwargs(a__    )

            __snake_case       , __snake_case			:Tuple          =						cls.get_config_dict(a__			,						**a__    )

            # get the text config dict if we are loading from AlignConfig
            if config_dict.get("""model_type"""    ) == "align":
                 __snake_case			:Any          =						config_dict["""text_config"""]

            if "model_type" in config_dict and hasattr(cls			,						"""model_type"""    ) and config_dict["model_type"] != cls.model_type:
                 logger.warning(
                     F'''You are using a model of type {config_dict['model_type']} to instantiate a model of type '''
                     F'''{cls.model_type}. This is not supported for all configurations of models and can yield errors.'''    )

            return cls.from_dict(a__			,						**a__    )




class 			snake_case__   (							lowercase_):

       '''simple docstring'''
       lowerCamelCase							:				Optional[int] 				= "align_vision_model"



       def __init__(							self			,						a__ = 3			,						a__ = 6_00			,						a__ = 2.0			,						a__ = 3.1			,						a__ = 8			,						a__ = [3, 3, 5, 3, 5, 5, 3]			,						a__ = [32, 16, 24, 40, 80, 1_12, 1_92]			,						a__ = [16, 24, 40, 80, 1_12, 1_92, 3_20]			,						a__ = []			,						a__ = [1, 2, 2, 2, 1, 2, 1]			,						a__ = [1, 2, 2, 3, 3, 4, 1]			,						a__ = [1, 6, 6, 6, 6, 6, 6]			,						a__ = 0.25			,						a__ = "swish"			,						a__ = 25_60			,						a__ = "mean"			,						a__ = 0.02			,						a__ = 0.0_01			,						a__ = 0.99			,						a__ = 0.2			,						**a__			,						) ->      List[Any]:






            '''simple docstring'''





            super().__init__(**a__    )

            __snake_case			:Union[str, Any]          =						num_channels
            __snake_case			:List[str]          =						image_size
            __snake_case			:int          =						width_coefficient
            __snake_case			:int          =						depth_coefficient
            __snake_case			:List[Any]          =						depth_divisor
            __snake_case			:Any          =						kernel_sizes
            __snake_case			:Optional[int]          =						in_channels
            __snake_case			:Optional[int]          =						out_channels
            __snake_case			:int          =						depthwise_padding
            __snake_case			:List[str]          =						strides
            __snake_case			:Union[str, Any]          =						num_block_repeats
            __snake_case			:Dict          =						expand_ratios
            __snake_case			:Union[str, Any]          =						squeeze_expansion_ratio
            __snake_case			:Any          =						hidden_act
            __snake_case			:Optional[Any]          =						hidden_dim
            __snake_case			:Union[str, Any]          =						pooling_type
            __snake_case			:Union[str, Any]          =						initializer_range
            __snake_case			:Optional[Any]          =						batch_norm_eps
            __snake_case			:List[Any]          =						batch_norm_momentum
            __snake_case			:Optional[int]          =						drop_connect_rate
            __snake_case			:Union[str, Any]          =						sum(a__    ) * 4



       @classmethod
       def 					__lowercase						(							cls			,						a__			,						**a__    ) ->      "PretrainedConfig":






            '''simple docstring'''





            cls._set_token_in_kwargs(a__    )

            __snake_case       , __snake_case			:int          =						cls.get_config_dict(a__			,						**a__    )

            # get the vision config dict if we are loading from AlignConfig
            if config_dict.get("""model_type"""    ) == "align":
                 __snake_case			:str          =						config_dict["""vision_config"""]

            if "model_type" in config_dict and hasattr(cls			,						"""model_type"""    ) and config_dict["model_type"] != cls.model_type:
                 logger.warning(
                     F'''You are using a model of type {config_dict['model_type']} to instantiate a model of type '''
                     F'''{cls.model_type}. This is not supported for all configurations of models and can yield errors.'''    )

            return cls.from_dict(a__			,						**a__    )




class 			snake_case__   (							lowercase_):

       '''simple docstring'''
       lowerCamelCase							:				str 				= "align"
       lowerCamelCase							:				Union[str, Any] 				= True



       def __init__(							self			,						a__=None			,						a__=None			,						a__=6_40			,						a__=1.0			,						a__=0.02			,						**a__			,						) ->      Dict:






            '''simple docstring'''





            super().__init__(**a__    )

            if text_config is None:
                 __snake_case			:Union[str, Any]          =						{}
                 logger.info("""text_config is None. Initializing the AlignTextConfig with default values."""    )

            if vision_config is None:
                 __snake_case			:str          =						{}
                 logger.info("""vision_config is None. Initializing the AlignVisionConfig with default values."""    )

            __snake_case			:List[Any]          =						AlignTextConfig(**a__    )
            __snake_case			:Tuple          =						AlignVisionConfig(**a__    )

            __snake_case			:Tuple          =						projection_dim
            __snake_case			:int          =						temperature_init_value
            __snake_case			:Any          =						initializer_range



       @classmethod
       def 					__lowercase						(							cls			,						a__			,						a__			,						**a__    ) ->      str:






            '''simple docstring'''





            return cls(text_config=text_config.to_dict()			,						vision_config=vision_config.to_dict()			,						**a__    )



       def 					__lowercase						(							self    ) ->      Dict:






            '''simple docstring'''





            __snake_case			:Optional[Any]          =						copy.deepcopy(self.__dict__    )
            __snake_case			:Dict          =						self.text_config.to_dict()
            __snake_case			:Union[str, Any]          =						self.vision_config.to_dict()
            __snake_case			:List[Any]          =						self.__class__.model_type
            return output


