


'''simple docstring'''



from typing import Optional

import numpy as np
import torch
from torch import nn
from transformers import GPTaConfig, GPTaLMHeadModel
from transformers.modeling_utils import ModuleUtilsMixin

from ...configuration_utils import ConfigMixin, register_to_config
from ...models import ModelMixin





class   UpperCAmelCase_    (_UpperCAmelCase   ,						_UpperCAmelCase   ,						_UpperCAmelCase    ):





    """simple docstring"""






    lowerCamelCase    :   Any					=     [r'h\.\d+\.attn\.bias', r'h\.\d+\.attn\.masked_bias']







    @register_to_config
    def __init__(				self     ,	SCREAMING_SNAKE_CASE_     ,	SCREAMING_SNAKE_CASE_     ,	SCREAMING_SNAKE_CASE_ = None     ,	SCREAMING_SNAKE_CASE_ = 5_02_57     ,	SCREAMING_SNAKE_CASE_ = 10_24     ,	SCREAMING_SNAKE_CASE_ = 7_68     ,	SCREAMING_SNAKE_CASE_ = 12     ,	SCREAMING_SNAKE_CASE_ = 12     ,	SCREAMING_SNAKE_CASE_ = None     ,	SCREAMING_SNAKE_CASE_ = "gelu_new"     ,	SCREAMING_SNAKE_CASE_ = 0.1     ,	SCREAMING_SNAKE_CASE_ = 0.1     ,	SCREAMING_SNAKE_CASE_ = 0.1     ,	SCREAMING_SNAKE_CASE_ = 1E-5     ,	SCREAMING_SNAKE_CASE_ = 0.0_2     ,	SCREAMING_SNAKE_CASE_ = True     ,	SCREAMING_SNAKE_CASE_ = True     ,	SCREAMING_SNAKE_CASE_ = False     ,	SCREAMING_SNAKE_CASE_ = False     ,	)							->	Tuple:
         super().__init__()

         __lowerCamelCase  :      int														=						prefix_length

         if prefix_inner_dim != n_embd and prefix_hidden_dim is None:
              raise ValueError(
                  f'`prefix_hidden_dim` cannot be `None` when `prefix_inner_dim`: {prefix_hidden_dim} and'
                  f' `n_embd`: {n_embd} are not equal.'      )

         __lowerCamelCase  :      List[Any]														=						prefix_inner_dim
         __lowerCamelCase  :      Any														=						prefix_hidden_dim

         __lowerCamelCase  :      Dict														=						(
             nn.Linear(self.prefix_inner_dim     ,	self.prefix_hidden_dim      )
             if self.prefix_hidden_dim is not None
             else nn.Identity()
         )
         __lowerCamelCase  :      Dict														=						(
             nn.Linear(self.prefix_hidden_dim     ,	SCREAMING_SNAKE_CASE_      ) if self.prefix_hidden_dim is not None else nn.Identity()
         )

         __lowerCamelCase  :      str														=						GPTaConfig(
             vocab_size=SCREAMING_SNAKE_CASE_     ,	n_positions=SCREAMING_SNAKE_CASE_     ,	n_embd=SCREAMING_SNAKE_CASE_     ,	n_layer=SCREAMING_SNAKE_CASE_     ,	n_head=SCREAMING_SNAKE_CASE_     ,	n_inner=SCREAMING_SNAKE_CASE_     ,	activation_function=SCREAMING_SNAKE_CASE_     ,	resid_pdrop=SCREAMING_SNAKE_CASE_     ,	embd_pdrop=SCREAMING_SNAKE_CASE_     ,	attn_pdrop=SCREAMING_SNAKE_CASE_     ,	layer_norm_epsilon=SCREAMING_SNAKE_CASE_     ,	initializer_range=SCREAMING_SNAKE_CASE_     ,	scale_attn_weights=SCREAMING_SNAKE_CASE_     ,	use_cache=SCREAMING_SNAKE_CASE_     ,	scale_attn_by_inverse_layer_idx=SCREAMING_SNAKE_CASE_     ,	reorder_and_upcast_attn=SCREAMING_SNAKE_CASE_     ,	)
         __lowerCamelCase  :      Dict														=						GPTaLMHeadModel(SCREAMING_SNAKE_CASE_      )







    def  lowercase_							(				self     ,	SCREAMING_SNAKE_CASE_     ,	SCREAMING_SNAKE_CASE_     ,	SCREAMING_SNAKE_CASE_ = None     ,	SCREAMING_SNAKE_CASE_ = None     ,	)							->	List[str]:
         __lowerCamelCase  :      Any														=						self.transformer.transformer.wte(SCREAMING_SNAKE_CASE_      )
         __lowerCamelCase  :      Tuple														=						self.encode_prefix(SCREAMING_SNAKE_CASE_      )
         __lowerCamelCase  :      List[str]														=						self.decode_prefix(SCREAMING_SNAKE_CASE_      )
         __lowerCamelCase  :      Any														=						torch.cat((prefix_embeds, embedding_text)     ,	dim=1      )

         if labels is not None:
              __lowerCamelCase  :      Union[str, Any]														=						self.get_dummy_token(input_ids.shape[0]     ,	input_ids.device      )
              __lowerCamelCase  :      Optional[Any]														=						torch.cat((dummy_token, input_ids)     ,	dim=1      )
         __lowerCamelCase  :      Union[str, Any]														=						self.transformer(inputs_embeds=SCREAMING_SNAKE_CASE_     ,	labels=SCREAMING_SNAKE_CASE_     ,	attention_mask=SCREAMING_SNAKE_CASE_      )
         if self.prefix_hidden_dim is not None:
              return out, hidden
         else:
              return out







    def  lowercase_							(				self     ,	SCREAMING_SNAKE_CASE_     ,	SCREAMING_SNAKE_CASE_      )							->	torch.Tensor:
         return torch.zeros(SCREAMING_SNAKE_CASE_     ,	self.prefix_length     ,	dtype=torch.intaa     ,	device=SCREAMING_SNAKE_CASE_      )







    def  lowercase_							(				self     ,	SCREAMING_SNAKE_CASE_      )							->	List[Any]:
         return self.encode_prefix(SCREAMING_SNAKE_CASE_      )







    @torch.no_grad()
    def  lowercase_							(				self     ,	SCREAMING_SNAKE_CASE_     ,	SCREAMING_SNAKE_CASE_     ,	SCREAMING_SNAKE_CASE_      )							->	List[Any]:
         __lowerCamelCase  :      int														=						torch.split(SCREAMING_SNAKE_CASE_     ,	1     ,	dim=0      )
         __lowerCamelCase  :      List[str]														=						[]
         __lowerCamelCase  :      Tuple														=						[]
         for feature in features:
              __lowerCamelCase  :      List[str]														=						self.decode_prefix(feature.to(SCREAMING_SNAKE_CASE_      )      )  # back to the clip feature
              # Only support beam search for now
              __lowerCamelCase							,			__lowerCamelCase  :      int														=						self.generate_beam(
                  input_embeds=SCREAMING_SNAKE_CASE_     ,	device=SCREAMING_SNAKE_CASE_     ,	eos_token_id=SCREAMING_SNAKE_CASE_      )
              generated_tokens.append(output_tokens[0]      )
              generated_seq_lengths.append(seq_lengths[0]      )
         __lowerCamelCase  :      Optional[Any]														=						torch.stack(SCREAMING_SNAKE_CASE_      )
         __lowerCamelCase  :      Optional[int]														=						torch.stack(SCREAMING_SNAKE_CASE_      )
         return generated_tokens, generated_seq_lengths







    @torch.no_grad()
    def  lowercase_							(				self     ,	SCREAMING_SNAKE_CASE_=None     ,	SCREAMING_SNAKE_CASE_=None     ,	SCREAMING_SNAKE_CASE_=None     ,	SCREAMING_SNAKE_CASE_ = 5     ,	SCREAMING_SNAKE_CASE_ = 67     ,	SCREAMING_SNAKE_CASE_ = 1.0     ,	SCREAMING_SNAKE_CASE_ = None     ,	)							->	Union[str, Any]:
         __lowerCamelCase  :      Dict														=						eos_token_id
         __lowerCamelCase  :      Dict														=						None
         __lowerCamelCase  :      List[str]														=						None
         __lowerCamelCase  :      Tuple														=						torch.ones(SCREAMING_SNAKE_CASE_     ,	device=SCREAMING_SNAKE_CASE_     ,	dtype=torch.int      )
         __lowerCamelCase  :      Dict														=						torch.zeros(SCREAMING_SNAKE_CASE_     ,	device=SCREAMING_SNAKE_CASE_     ,	dtype=torch.bool      )

         if input_embeds is not None:
              __lowerCamelCase  :      Any														=						input_embeds
         else:
              __lowerCamelCase  :      Dict														=						self.transformer.transformer.wte(SCREAMING_SNAKE_CASE_      )

         for i in range(SCREAMING_SNAKE_CASE_      ):
              __lowerCamelCase  :      Tuple														=						self.transformer(inputs_embeds=SCREAMING_SNAKE_CASE_      )
              __lowerCamelCase  :      int														=						outputs.logits
              __lowerCamelCase  :      Tuple														=						logits[:, -1, :] / (temperature if temperature > 0 else 1.0)
              __lowerCamelCase  :      Optional[int]														=						logits.softmax(-1      ).log()

              if scores is None:
                   __lowerCamelCase							,			__lowerCamelCase  :      Dict														=						logits.topk(SCREAMING_SNAKE_CASE_     ,	-1      )
                   __lowerCamelCase  :      Union[str, Any]														=						generated.expand(SCREAMING_SNAKE_CASE_     ,	*generated.shape[1:]      )
                   __lowerCamelCase							,			__lowerCamelCase  :      Union[str, Any]														=						next_tokens.permute(1     ,	0      ), scores.squeeze(0      )
                   if tokens is None:
                        __lowerCamelCase  :      Optional[int]														=						next_tokens
                   else:
                        __lowerCamelCase  :      int														=						tokens.expand(SCREAMING_SNAKE_CASE_     ,	*tokens.shape[1:]      )
                        __lowerCamelCase  :      Any														=						torch.cat((tokens, next_tokens)     ,	dim=1      )
              else:
                   __lowerCamelCase  :      Optional[int]														=						-float(np.inf      )
                   __lowerCamelCase  :      Union[str, Any]														=						0
                   __lowerCamelCase  :      str														=						scores[:, None] + logits
                   seq_lengths[~is_stopped] += 1
                   __lowerCamelCase  :      int														=						scores_sum / seq_lengths[:, None]
                   __lowerCamelCase							,			__lowerCamelCase  :      Optional[int]														=						scores_sum_average.view(-1      ).topk(SCREAMING_SNAKE_CASE_     ,	-1      )
                   __lowerCamelCase  :      int														=						next_tokens // scores_sum.shape[1]
                   __lowerCamelCase  :      List[str]														=						seq_lengths[next_tokens_source]
                   __lowerCamelCase  :      Optional[int]														=						next_tokens % scores_sum.shape[1]
                   __lowerCamelCase  :      Tuple														=						next_tokens.unsqueeze(1      )
                   __lowerCamelCase  :      List[str]														=						tokens[next_tokens_source]
                   __lowerCamelCase  :      Tuple														=						torch.cat((tokens, next_tokens)     ,	dim=1      )
                   __lowerCamelCase  :      int														=						generated[next_tokens_source]
                   __lowerCamelCase  :      Optional[Any]														=						scores_sum_average * seq_lengths
                   __lowerCamelCase  :      str														=						is_stopped[next_tokens_source]

              __lowerCamelCase  :      List[Any]														=						self.transformer.transformer.wte(next_tokens.squeeze()      ).view(generated.shape[0]     ,	1     ,	-1      )
              __lowerCamelCase  :      Any														=						torch.cat((generated, next_token_embed)     ,	dim=1      )
              __lowerCamelCase  :      Dict														=						is_stopped + next_tokens.eq(SCREAMING_SNAKE_CASE_      ).squeeze()
              if is_stopped.all():
                   break

         __lowerCamelCase  :      Tuple														=						scores / seq_lengths
         __lowerCamelCase  :      Tuple														=						scores.argsort(descending=SCREAMING_SNAKE_CASE_      )
         # tokens tensors are already padded to max_seq_length
         __lowerCamelCase  :      Union[str, Any]														=						[tokens[i] for i in order]
         __lowerCamelCase  :      List[Any]														=						torch.stack(SCREAMING_SNAKE_CASE_     ,	dim=0      )
         __lowerCamelCase  :      int														=						torch.tensor([seq_lengths[i] for i in order]     ,	dtype=seq_lengths.dtype      )
         return output_texts, seq_lengths










'''simple docstring'''



import inspect
import unittest

import numpy as np

from transformers import ViTConfig, is_flax_available
from transformers.testing_utils import require_flax, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_flax_common import FlaxModelTesterMixin, floats_tensor


if is_flax_available():
      import jax

      from transformers.models.vit.modeling_flax_vit import FlaxViTForImageClassification, FlaxViTModel





class   UpperCAmelCase_    (unittest.TestCase    ):





    """simple docstring"""






    def __init__(				self     ,	SCREAMING_SNAKE_CASE_     ,	SCREAMING_SNAKE_CASE_=13     ,	SCREAMING_SNAKE_CASE_=30     ,	SCREAMING_SNAKE_CASE_=2     ,	SCREAMING_SNAKE_CASE_=3     ,	SCREAMING_SNAKE_CASE_=True     ,	SCREAMING_SNAKE_CASE_=True     ,	SCREAMING_SNAKE_CASE_=32     ,	SCREAMING_SNAKE_CASE_=5     ,	SCREAMING_SNAKE_CASE_=4     ,	SCREAMING_SNAKE_CASE_=37     ,	SCREAMING_SNAKE_CASE_="gelu"     ,	SCREAMING_SNAKE_CASE_=0.1     ,	SCREAMING_SNAKE_CASE_=0.1     ,	SCREAMING_SNAKE_CASE_=10     ,	SCREAMING_SNAKE_CASE_=0.0_2     ,	)							->	Optional[int]:
         __lowerCamelCase  :      Optional[int]														=						parent
         __lowerCamelCase  :      Dict														=						batch_size
         __lowerCamelCase  :      int														=						image_size
         __lowerCamelCase  :      List[str]														=						patch_size
         __lowerCamelCase  :      Optional[int]														=						num_channels
         __lowerCamelCase  :      Any														=						is_training
         __lowerCamelCase  :      Dict														=						use_labels
         __lowerCamelCase  :      List[Any]														=						hidden_size
         __lowerCamelCase  :      List[Any]														=						num_hidden_layers
         __lowerCamelCase  :      Optional[Any]														=						num_attention_heads
         __lowerCamelCase  :      Dict														=						intermediate_size
         __lowerCamelCase  :      Union[str, Any]														=						hidden_act
         __lowerCamelCase  :      Optional[int]														=						hidden_dropout_prob
         __lowerCamelCase  :      Tuple														=						attention_probs_dropout_prob
         __lowerCamelCase  :      str														=						type_sequence_label_size
         __lowerCamelCase  :      List[str]														=						initializer_range

         # in ViT, the seq length equals the number of patches + 1 (we add 1 for the [CLS] token)
         __lowerCamelCase  :      str														=						(image_size // patch_size) ** 2
         __lowerCamelCase  :      Optional[int]														=						num_patches + 1







    def  lowercase_							(				self      )							->	Union[str, Any]:
         __lowerCamelCase  :      int														=						floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size]      )

         __lowerCamelCase  :      Optional[int]														=						ViTConfig(
             image_size=self.image_size     ,	patch_size=self.patch_size     ,	num_channels=self.num_channels     ,	hidden_size=self.hidden_size     ,	num_hidden_layers=self.num_hidden_layers     ,	num_attention_heads=self.num_attention_heads     ,	intermediate_size=self.intermediate_size     ,	hidden_act=self.hidden_act     ,	hidden_dropout_prob=self.hidden_dropout_prob     ,	attention_probs_dropout_prob=self.attention_probs_dropout_prob     ,	is_decoder=SCREAMING_SNAKE_CASE_     ,	initializer_range=self.initializer_range     ,	)

         return config, pixel_values







    def  lowercase_							(				self     ,	SCREAMING_SNAKE_CASE_     ,	SCREAMING_SNAKE_CASE_      )							->	Union[str, Any]:
         __lowerCamelCase  :      Union[str, Any]														=						FlaxViTModel(config=SCREAMING_SNAKE_CASE_      )
         __lowerCamelCase  :      str														=						model(SCREAMING_SNAKE_CASE_      )
         # expected sequence length = num_patches + 1 (we add 1 for the [CLS] token)
         __lowerCamelCase  :      str														=						(self.image_size, self.image_size)
         __lowerCamelCase  :      str														=						(self.patch_size, self.patch_size)
         __lowerCamelCase  :      Any														=						(image_size[1] // patch_size[1]) * (image_size[0] // patch_size[0])
         self.parent.assertEqual(result.last_hidden_state.shape     ,	(self.batch_size, num_patches + 1, self.hidden_size)      )







    def  lowercase_							(				self     ,	SCREAMING_SNAKE_CASE_     ,	SCREAMING_SNAKE_CASE_      )							->	str:
         __lowerCamelCase  :      Tuple														=						self.type_sequence_label_size
         __lowerCamelCase  :      Any														=						FlaxViTForImageClassification(config=SCREAMING_SNAKE_CASE_      )
         __lowerCamelCase  :      Dict														=						model(SCREAMING_SNAKE_CASE_      )
         self.parent.assertEqual(result.logits.shape     ,	(self.batch_size, self.type_sequence_label_size)      )

         # test greyscale images
         __lowerCamelCase  :      List[str]														=						1
         __lowerCamelCase  :      List[Any]														=						FlaxViTForImageClassification(SCREAMING_SNAKE_CASE_      )

         __lowerCamelCase  :      str														=						floats_tensor([self.batch_size, 1, self.image_size, self.image_size]      )
         __lowerCamelCase  :      List[Any]														=						model(SCREAMING_SNAKE_CASE_      )







    def  lowercase_							(				self      )							->	Optional[int]:
         __lowerCamelCase  :      List[Any]														=						self.prepare_config_and_inputs()
         (
             (
             __lowerCamelCase
         )							,			(
             __lowerCamelCase
         )							,			
         )  :      int														=						config_and_inputs
         __lowerCamelCase  :      Union[str, Any]														=						{'pixel_values': pixel_values}
         return config, inputs_dict







@require_flax
class   UpperCAmelCase_    (_UpperCAmelCase   ,						unittest.TestCase    ):





    """simple docstring"""






    lowerCamelCase    :   str					=     (FlaxViTModel, FlaxViTForImageClassification) if is_flax_available() else ()







    def  lowercase_							(				self      )							->	None:
         __lowerCamelCase  :      str														=						FlaxViTModelTester(self      )
         __lowerCamelCase  :      Union[str, Any]														=						ConfigTester(self     ,	config_class=SCREAMING_SNAKE_CASE_     ,	has_text_modality=SCREAMING_SNAKE_CASE_     ,	hidden_size=37      )







    def  lowercase_							(				self      )							->	List[Any]:
         self.config_tester.run_common_tests()







    def  lowercase_							(				self      )							->	Optional[Any]:
         __lowerCamelCase  :      Optional[int]														=						self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_model(*SCREAMING_SNAKE_CASE_      )







    def  lowercase_							(				self      )							->	Any:
         __lowerCamelCase  :      List[Any]														=						self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_image_classification(*SCREAMING_SNAKE_CASE_      )







    def  lowercase_							(				self      )							->	Optional[Any]:
         __lowerCamelCase							,			__lowerCamelCase  :      Optional[int]														=						self.model_tester.prepare_config_and_inputs_for_common()

         for model_class in self.all_model_classes:
              __lowerCamelCase  :      Optional[Any]														=						model_class(SCREAMING_SNAKE_CASE_      )
              __lowerCamelCase  :      Dict														=						inspect.signature(model.__call__      )
              # signature.parameters is an OrderedDict => so arg_names order is deterministic
              __lowerCamelCase  :      List[str]														=						[*signature.parameters.keys()]

              __lowerCamelCase  :      Optional[int]														=						['pixel_values']
              self.assertListEqual(arg_names[:1]     ,	SCREAMING_SNAKE_CASE_      )







    def  lowercase_							(				self      )							->	Any:
         __lowerCamelCase							,			__lowerCamelCase  :      Dict														=						self.model_tester.prepare_config_and_inputs_for_common()

         for model_class in self.all_model_classes:
              with self.subTest(model_class.__name__      ):
                   __lowerCamelCase  :      List[Any]														=						self._prepare_for_class(SCREAMING_SNAKE_CASE_     ,	SCREAMING_SNAKE_CASE_      )
                   __lowerCamelCase  :      Optional[int]														=						model_class(SCREAMING_SNAKE_CASE_      )

                   @jax.jit
                   def model_jitted(SCREAMING_SNAKE_CASE_     ,	**SCREAMING_SNAKE_CASE_      ):
                        return model(pixel_values=SCREAMING_SNAKE_CASE_     ,	**SCREAMING_SNAKE_CASE_      )

                   with self.subTest('JIT Enabled'      ):
                        __lowerCamelCase  :      Optional[int]														=						model_jitted(**SCREAMING_SNAKE_CASE_      ).to_tuple()

                   with self.subTest('JIT Disabled'      ):
                        with jax.disable_jit():
                             __lowerCamelCase  :      Union[str, Any]														=						model_jitted(**SCREAMING_SNAKE_CASE_      ).to_tuple()

                   self.assertEqual(len(SCREAMING_SNAKE_CASE_      )     ,	len(SCREAMING_SNAKE_CASE_      )      )
                   for jitted_output, output in zip(SCREAMING_SNAKE_CASE_     ,	SCREAMING_SNAKE_CASE_      ):
                        self.assertEqual(jitted_output.shape     ,	output.shape      )







    @slow
    def  lowercase_							(				self      )							->	List[Any]:
         for model_class_name in self.all_model_classes:
              __lowerCamelCase  :      Union[str, Any]														=						model_class_name.from_pretrained('google/vit-base-patch16-224'      )
              __lowerCamelCase  :      Union[str, Any]														=						model(np.ones((1, 3, 2_24, 2_24)      )      )
              self.assertIsNotNone(SCREAMING_SNAKE_CASE_      )






