





"""simple docstring"""






import copy
import os
from typing import TYPE_CHECKING, List, Union


if TYPE_CHECKING:
		pass

from ...configuration_utils import PretrainedConfig
from ...utils import logging


A			      =	logging.get_logger(__name__)

A			      =	{
    '''kakaobrain/align-base''': '''https://huggingface.co/kakaobrain/align-base/resolve/main/config.json''',
}




class 					__lowercase      (      _UpperCamelCase				):







					'''simple docstring'''




					__lowerCAmelCase	=	'''align_text_model'''







					def __init__(    self						,      _UpperCAmelCase=30522						,      _UpperCAmelCase=768						,      _UpperCAmelCase=12						,      _UpperCAmelCase=12						,      _UpperCAmelCase=3072						,      _UpperCAmelCase="gelu"						,      _UpperCAmelCase=0.1						,      _UpperCAmelCase=0.1						,      _UpperCAmelCase=512						,      _UpperCAmelCase=2						,      _UpperCAmelCase=0.0_2						,      _UpperCAmelCase=1e-1_2						,      _UpperCAmelCase=0						,      _UpperCAmelCase="absolute"						,      _UpperCAmelCase=True						,      **_UpperCAmelCase						,      ):
									super().__init__(**_UpperCAmelCase  )

									__a			:    int      							=	vocab_size
									__a			:    Optional[int]      							=	hidden_size
									__a			:    Dict      							=	num_hidden_layers
									__a			:    List[Any]      							=	num_attention_heads
									__a			:    Optional[int]      							=	hidden_act
									__a			:    List[Any]      							=	intermediate_size
									__a			:    List[Any]      							=	hidden_dropout_prob
									__a			:    List[str]      							=	attention_probs_dropout_prob
									__a			:    Optional[int]      							=	max_position_embeddings
									__a			:    List[str]      							=	type_vocab_size
									__a			:    Tuple      							=	initializer_range
									__a			:    Dict      							=	layer_norm_eps
									__a			:    Any      							=	position_embedding_type
									__a			:    Dict      							=	use_cache
									__a			:    Dict      							=	pad_token_id







					@classmethod
					def 			_lowerCamelCase		(    cls						,      _UpperCAmelCase						,      **_UpperCAmelCase  ):
									cls._set_token_in_kwargs(_UpperCAmelCase  )

									__a							,    __a			:    List[str]      							=	cls.get_config_dict(_UpperCAmelCase						,      **_UpperCAmelCase  )

									# get the text config dict if we are loading from AlignConfig
									if config_dict.get('''model_type'''  ) == "align":
													__a			:    Dict      							=	config_dict['''text_config''']

									if "model_type" in config_dict and hasattr(cls						,      '''model_type'''  ) and config_dict["model_type"] != cls.model_type:
													logger.warning(
													    f"""You are using a model of type {config_dict["model_type"]} to instantiate a model of type """
													    f"""{cls.model_type}. This is not supported for all configurations of models and can yield errors."""  )

									return cls.from_dict(_UpperCAmelCase						,      **_UpperCAmelCase  )






class 					__lowercase      (      _UpperCamelCase				):







					'''simple docstring'''




					__lowerCAmelCase	=	'''align_vision_model'''







					def __init__(    self						,      _UpperCAmelCase = 3						,      _UpperCAmelCase = 600						,      _UpperCAmelCase = 2.0						,      _UpperCAmelCase = 3.1						,      _UpperCAmelCase = 8						,      _UpperCAmelCase = [3, 3, 5, 3, 5, 5, 3]						,      _UpperCAmelCase = [32, 16, 24, 40, 80, 112, 192]						,      _UpperCAmelCase = [16, 24, 40, 80, 112, 192, 320]						,      _UpperCAmelCase = []						,      _UpperCAmelCase = [1, 2, 2, 2, 1, 2, 1]						,      _UpperCAmelCase = [1, 2, 2, 3, 3, 4, 1]						,      _UpperCAmelCase = [1, 6, 6, 6, 6, 6, 6]						,      _UpperCAmelCase = 0.2_5						,      _UpperCAmelCase = "swish"						,      _UpperCAmelCase = 2560						,      _UpperCAmelCase = "mean"						,      _UpperCAmelCase = 0.0_2						,      _UpperCAmelCase = 0.0_0_1						,      _UpperCAmelCase = 0.9_9						,      _UpperCAmelCase = 0.2						,      **_UpperCAmelCase						,      ):
									super().__init__(**_UpperCAmelCase  )

									__a			:    Tuple      							=	num_channels
									__a			:    str      							=	image_size
									__a			:    List[Any]      							=	width_coefficient
									__a			:    Optional[int]      							=	depth_coefficient
									__a			:    Union[str, Any]      							=	depth_divisor
									__a			:    int      							=	kernel_sizes
									__a			:    Dict      							=	in_channels
									__a			:    List[str]      							=	out_channels
									__a			:    Any      							=	depthwise_padding
									__a			:    str      							=	strides
									__a			:    Optional[Any]      							=	num_block_repeats
									__a			:    Optional[Any]      							=	expand_ratios
									__a			:    Any      							=	squeeze_expansion_ratio
									__a			:    int      							=	hidden_act
									__a			:    Union[str, Any]      							=	hidden_dim
									__a			:    Union[str, Any]      							=	pooling_type
									__a			:    Tuple      							=	initializer_range
									__a			:    List[str]      							=	batch_norm_eps
									__a			:    List[Any]      							=	batch_norm_momentum
									__a			:    Union[str, Any]      							=	drop_connect_rate
									__a			:    List[Any]      							=	sum(_UpperCAmelCase  ) * 4







					@classmethod
					def 			_lowerCamelCase		(    cls						,      _UpperCAmelCase						,      **_UpperCAmelCase  ):
									cls._set_token_in_kwargs(_UpperCAmelCase  )

									__a							,    __a			:    Optional[Any]      							=	cls.get_config_dict(_UpperCAmelCase						,      **_UpperCAmelCase  )

									# get the vision config dict if we are loading from AlignConfig
									if config_dict.get('''model_type'''  ) == "align":
													__a			:    Optional[Any]      							=	config_dict['''vision_config''']

									if "model_type" in config_dict and hasattr(cls						,      '''model_type'''  ) and config_dict["model_type"] != cls.model_type:
													logger.warning(
													    f"""You are using a model of type {config_dict["model_type"]} to instantiate a model of type """
													    f"""{cls.model_type}. This is not supported for all configurations of models and can yield errors."""  )

									return cls.from_dict(_UpperCAmelCase						,      **_UpperCAmelCase  )






class 					__lowercase      (      _UpperCamelCase				):







					'''simple docstring'''




					__lowerCAmelCase	=	'''align'''
					__lowerCAmelCase	=	True







					def __init__(    self						,      _UpperCAmelCase=None						,      _UpperCAmelCase=None						,      _UpperCAmelCase=640						,      _UpperCAmelCase=1.0						,      _UpperCAmelCase=0.0_2						,      **_UpperCAmelCase						,      ):
									super().__init__(**_UpperCAmelCase  )

									if text_config is None:
													__a			:    Dict      							=	{}
													logger.info('''text_config is None. Initializing the AlignTextConfig with default values.'''  )

									if vision_config is None:
													__a			:    Any      							=	{}
													logger.info('''vision_config is None. Initializing the AlignVisionConfig with default values.'''  )

									__a			:    Any      							=	AlignTextConfig(**_UpperCAmelCase  )
									__a			:    Any      							=	AlignVisionConfig(**_UpperCAmelCase  )

									__a			:    Optional[int]      							=	projection_dim
									__a			:    Union[str, Any]      							=	temperature_init_value
									__a			:    int      							=	initializer_range







					@classmethod
					def 			_lowerCamelCase		(    cls						,      _UpperCAmelCase						,      _UpperCAmelCase						,      **_UpperCAmelCase  ):
									return cls(text_config=text_config.to_dict()						,      vision_config=vision_config.to_dict()						,      **_UpperCAmelCase  )







					def 			_lowerCamelCase		(    self  ):
									__a			:    Union[str, Any]      							=	copy.deepcopy(self.__dict__  )
									__a			:    Tuple      							=	self.text_config.to_dict()
									__a			:    Union[str, Any]      							=	self.vision_config.to_dict()
									__a			:    int      							=	self.__class__.model_type
									return output






import torch
from torch import nn



class 					__snake_case	(  nn.Module    ):


    '''simple docstring'''







    def __init__(       self   ,				A_   ,				A_   ,				A_   ,				A_   ,				A_=1   ,				A_=False     ):





        '''simple docstring'''

        super().__init__()

        SCREAMING_SNAKE_CASE__											=			n_token
        SCREAMING_SNAKE_CASE__											=			d_embed
        SCREAMING_SNAKE_CASE__											=			d_proj

        SCREAMING_SNAKE_CASE__											=			cutoffs + [n_token]
        SCREAMING_SNAKE_CASE__											=			[0] + self.cutoffs
        SCREAMING_SNAKE_CASE__											=			div_val

        SCREAMING_SNAKE_CASE__											=			self.cutoffs[0]
        SCREAMING_SNAKE_CASE__											=			len(self.cutoffs     ) - 1
        SCREAMING_SNAKE_CASE__											=			self.shortlist_size + self.n_clusters

        if self.n_clusters > 0:
            SCREAMING_SNAKE_CASE__											=			nn.Parameter(torch.zeros(self.n_clusters   ,				self.d_embed     )     )
            SCREAMING_SNAKE_CASE__											=			nn.Parameter(torch.zeros(self.n_clusters     )     )

        SCREAMING_SNAKE_CASE__											=			nn.ModuleList()
        SCREAMING_SNAKE_CASE__											=			nn.ParameterList()

        if div_val == 1:
            for i in range(len(self.cutoffs     )     ):
                if d_proj != d_embed:
                    self.out_projs.append(nn.Parameter(torch.FloatTensor(A_   ,				A_     )     )     )
                else:
                    self.out_projs.append(A_     )

            self.out_layers.append(nn.Linear(A_   ,				A_     )     )
        else:
            for i in range(len(self.cutoffs     )     ):
                SCREAMING_SNAKE_CASE__				,		SCREAMING_SNAKE_CASE__											=			self.cutoff_ends[i], self.cutoff_ends[i + 1]
                SCREAMING_SNAKE_CASE__											=			d_embed // (div_val**i)

                self.out_projs.append(nn.Parameter(torch.FloatTensor(A_   ,				A_     )     )     )

                self.out_layers.append(nn.Linear(A_   ,				r_idx - l_idx     )     )

        SCREAMING_SNAKE_CASE__											=			keep_order



    def  lowercase_					(       self   ,				A_   ,				A_   ,				A_   ,				A_     ):





        '''simple docstring'''

        if proj is None:
            SCREAMING_SNAKE_CASE__											=			nn.functional.linear(A_   ,				A_   ,				bias=A_     )
        else:
            # if CUDA_MAJOR <= 9 and CUDA_MINOR <= 1:
            SCREAMING_SNAKE_CASE__											=			nn.functional.linear(A_   ,				proj.t().contiguous()     )
            SCREAMING_SNAKE_CASE__											=			nn.functional.linear(A_   ,				A_   ,				bias=A_     )
            # else:
            #     logit = torch.einsum('bd,de,ev->bv', (hidden, proj, weight.t()))
            #     if bias is not None:
            #         logit = logit + bias

        return logit



    def  lowercase_					(       self   ,				A_   ,				A_=None   ,				A_=False     ):





        '''simple docstring'''

        if labels is not None:
            # Shift so that tokens < n predict n
            SCREAMING_SNAKE_CASE__											=			hidden[..., :-1, :].contiguous()
            SCREAMING_SNAKE_CASE__											=			labels[..., 1:].contiguous()
            SCREAMING_SNAKE_CASE__											=			hidden.view(-1   ,				hidden.size(-1     )     )
            SCREAMING_SNAKE_CASE__											=			labels.view(-1     )
            if hidden.size(0     ) != labels.size(0     ):
                raise RuntimeError('''Input and labels should have the same size in the batch dimension.'''     )
        else:
            SCREAMING_SNAKE_CASE__											=			hidden.view(-1   ,				hidden.size(-1     )     )

        if self.n_clusters == 0:
            SCREAMING_SNAKE_CASE__											=			self._compute_logit(A_   ,				self.out_layers[0].weight   ,				self.out_layers[0].bias   ,				self.out_projs[0]     )
            if labels is not None:
                SCREAMING_SNAKE_CASE__											=			labels != -1_00
                SCREAMING_SNAKE_CASE__											=			torch.zeros_like(A_   ,				dtype=hidden.dtype   ,				device=hidden.device     )
                SCREAMING_SNAKE_CASE__											=			(
                    -nn.functional.log_softmax(A_   ,				dim=-1     )[mask].gather(1   ,				labels[mask].unsqueeze(1     )     ).squeeze(1     )
                )
            else:
                SCREAMING_SNAKE_CASE__											=			nn.functional.log_softmax(A_   ,				dim=-1     )
        else:
            # construct weights and biases
            SCREAMING_SNAKE_CASE__				,		SCREAMING_SNAKE_CASE__											=			[], []
            for i in range(len(self.cutoffs     )     ):
                if self.div_val == 1:
                    SCREAMING_SNAKE_CASE__				,		SCREAMING_SNAKE_CASE__											=			self.cutoff_ends[i], self.cutoff_ends[i + 1]
                    SCREAMING_SNAKE_CASE__											=			self.out_layers[0].weight[l_idx:r_idx]
                    SCREAMING_SNAKE_CASE__											=			self.out_layers[0].bias[l_idx:r_idx]
                else:
                    SCREAMING_SNAKE_CASE__											=			self.out_layers[i].weight
                    SCREAMING_SNAKE_CASE__											=			self.out_layers[i].bias

                if i == 0:
                    SCREAMING_SNAKE_CASE__											=			torch.cat([weight_i, self.cluster_weight]   ,				dim=0     )
                    SCREAMING_SNAKE_CASE__											=			torch.cat([bias_i, self.cluster_bias]   ,				dim=0     )

                weights.append(A_     )
                biases.append(A_     )

            SCREAMING_SNAKE_CASE__				,		SCREAMING_SNAKE_CASE__				,		SCREAMING_SNAKE_CASE__											=			weights[0], biases[0], self.out_projs[0]

            SCREAMING_SNAKE_CASE__											=			self._compute_logit(A_   ,				A_   ,				A_   ,				A_     )
            SCREAMING_SNAKE_CASE__											=			nn.functional.log_softmax(A_   ,				dim=1     )

            if labels is None:
                SCREAMING_SNAKE_CASE__											=			hidden.new_empty((head_logit.size(0     ), self.n_token)     )
            else:
                SCREAMING_SNAKE_CASE__											=			torch.zeros_like(A_   ,				dtype=hidden.dtype   ,				device=hidden.device     )

            SCREAMING_SNAKE_CASE__											=			0
            SCREAMING_SNAKE_CASE__											=			[0] + self.cutoffs
            for i in range(len(A_     ) - 1     ):
                SCREAMING_SNAKE_CASE__				,		SCREAMING_SNAKE_CASE__											=			cutoff_values[i], cutoff_values[i + 1]

                if labels is not None:
                    SCREAMING_SNAKE_CASE__											=			(labels >= l_idx) & (labels < r_idx)
                    SCREAMING_SNAKE_CASE__											=			mask_i.nonzero().squeeze()

                    if indices_i.numel() == 0:
                        continue

                    SCREAMING_SNAKE_CASE__											=			labels.index_select(0   ,				A_     ) - l_idx
                    SCREAMING_SNAKE_CASE__											=			head_logprob.index_select(0   ,				A_     )
                    SCREAMING_SNAKE_CASE__											=			hidden.index_select(0   ,				A_     )
                else:
                    SCREAMING_SNAKE_CASE__											=			hidden

                if i == 0:
                    if labels is not None:
                        SCREAMING_SNAKE_CASE__											=			head_logprob_i.gather(1   ,				target_i[:, None]     ).squeeze(1     )
                    else:
                        SCREAMING_SNAKE_CASE__											=			head_logprob[:, : self.cutoffs[0]]
                else:
                    SCREAMING_SNAKE_CASE__				,		SCREAMING_SNAKE_CASE__				,		SCREAMING_SNAKE_CASE__											=			weights[i], biases[i], self.out_projs[i]

                    SCREAMING_SNAKE_CASE__											=			self._compute_logit(A_   ,				A_   ,				A_   ,				A_     )
                    SCREAMING_SNAKE_CASE__											=			nn.functional.log_softmax(A_   ,				dim=1     )
                    SCREAMING_SNAKE_CASE__											=			self.cutoffs[0] + i - 1  # No probability for the head cluster
                    if labels is not None:
                        SCREAMING_SNAKE_CASE__											=			head_logprob_i[:, cluster_prob_idx] + tail_logprob_i.gather(
                            1   ,				target_i[:, None]     ).squeeze(1     )
                    else:
                        SCREAMING_SNAKE_CASE__											=			head_logprob[:, cluster_prob_idx, None] + tail_logprob_i
                        SCREAMING_SNAKE_CASE__											=			logprob_i

                if labels is not None:
                    if (hasattr(self   ,				'''keep_order'''     ) and self.keep_order) or keep_order:
                        out.index_copy_(0   ,				A_   ,				-logprob_i     )
                    else:
                        out[offset : offset + logprob_i.size(0     )].copy_(-logprob_i     )
                    offset += logprob_i.size(0     )

        return out




    def  lowercase_					(       self   ,				A_     ):





        '''simple docstring'''

        if self.n_clusters == 0:
            SCREAMING_SNAKE_CASE__											=			self._compute_logit(A_   ,				self.out_layers[0].weight   ,				self.out_layers[0].bias   ,				self.out_projs[0]     )
            return nn.functional.log_softmax(A_   ,				dim=-1     )
        else:
            # construct weights and biases
            SCREAMING_SNAKE_CASE__				,		SCREAMING_SNAKE_CASE__											=			[], []
            for i in range(len(self.cutoffs     )     ):
                if self.div_val == 1:
                    SCREAMING_SNAKE_CASE__				,		SCREAMING_SNAKE_CASE__											=			self.cutoff_ends[i], self.cutoff_ends[i + 1]
                    SCREAMING_SNAKE_CASE__											=			self.out_layers[0].weight[l_idx:r_idx]
                    SCREAMING_SNAKE_CASE__											=			self.out_layers[0].bias[l_idx:r_idx]
                else:
                    SCREAMING_SNAKE_CASE__											=			self.out_layers[i].weight
                    SCREAMING_SNAKE_CASE__											=			self.out_layers[i].bias

                if i == 0:
                    SCREAMING_SNAKE_CASE__											=			torch.cat([weight_i, self.cluster_weight]   ,				dim=0     )
                    SCREAMING_SNAKE_CASE__											=			torch.cat([bias_i, self.cluster_bias]   ,				dim=0     )

                weights.append(A_     )
                biases.append(A_     )

            SCREAMING_SNAKE_CASE__				,		SCREAMING_SNAKE_CASE__				,		SCREAMING_SNAKE_CASE__											=			weights[0], biases[0], self.out_projs[0]
            SCREAMING_SNAKE_CASE__											=			self._compute_logit(A_   ,				A_   ,				A_   ,				A_     )

            SCREAMING_SNAKE_CASE__											=			hidden.new_empty((head_logit.size(0     ), self.n_token)     )
            SCREAMING_SNAKE_CASE__											=			nn.functional.log_softmax(A_   ,				dim=1     )

            SCREAMING_SNAKE_CASE__											=			[0] + self.cutoffs
            for i in range(len(A_     ) - 1     ):
                SCREAMING_SNAKE_CASE__				,		SCREAMING_SNAKE_CASE__											=			cutoff_values[i], cutoff_values[i + 1]

                if i == 0:
                    SCREAMING_SNAKE_CASE__											=			head_logprob[:, : self.cutoffs[0]]
                else:
                    SCREAMING_SNAKE_CASE__				,		SCREAMING_SNAKE_CASE__				,		SCREAMING_SNAKE_CASE__											=			weights[i], biases[i], self.out_projs[i]

                    SCREAMING_SNAKE_CASE__											=			self._compute_logit(A_   ,				A_   ,				A_   ,				A_     )
                    SCREAMING_SNAKE_CASE__											=			nn.functional.log_softmax(A_   ,				dim=1     )

                    SCREAMING_SNAKE_CASE__											=			head_logprob[:, -i] + tail_logprob_i
                    SCREAMING_SNAKE_CASE__											=			logprob_i

            return out
