
import torch
from torch import nn




class   __lowercase						(						nn.Module       ):






	def __init__(self				: Optional[int]      ,	snake_case				: Optional[int]      ,	snake_case				: Dict      ,	snake_case				: Union[str, Any]      ,	snake_case				: List[str]      ,	snake_case				: List[Any]=1      ,	snake_case				: int=False    ) ->			int:
							super().__init__()

							_lowercase       :	Optional[int]           =   n_token
							_lowercase       :	List[Any]           =   d_embed
							_lowercase       :	str           =   d_proj

							_lowercase       :	int           =   cutoffs + [n_token]
							_lowercase       :	Dict           =   [0] + self.cutoffs
							_lowercase       :	str           =   div_val

							_lowercase       :	Optional[Any]           =   self.cutoffs[0]
							_lowercase       :	List[str]           =   len(self.cutoffs    ) - 1
							_lowercase       :	List[str]           =   self.shortlist_size + self.n_clusters

							if self.n_clusters > 0:
													_lowercase       :	Tuple           =   nn.Parameter(torch.zeros(self.n_clusters      ,	self.d_embed    )    )
													_lowercase       :	Union[str, Any]           =   nn.Parameter(torch.zeros(self.n_clusters    )    )

							_lowercase       :	List[str]           =   nn.ModuleList()
							_lowercase       :	List[str]           =   nn.ParameterList()

							if div_val == 1:
													for i in range(len(self.cutoffs    )    ):
																			if d_proj != d_embed:
																									self.out_projs.append(nn.Parameter(torch.FloatTensor(snake_case      ,	snake_case    )    )    )
																			else:
																									self.out_projs.append(snake_case    )

													self.out_layers.append(nn.Linear(snake_case      ,	snake_case    )    )
							else:
													for i in range(len(self.cutoffs    )    ):
																			_lowercase      ,    _lowercase       :	Dict           =   self.cutoff_ends[i], self.cutoff_ends[i + 1]
																			_lowercase       :	Tuple           =   d_embed // (div_val**i)

																			self.out_projs.append(nn.Parameter(torch.FloatTensor(snake_case      ,	snake_case    )    )    )

																			self.out_layers.append(nn.Linear(snake_case      ,	r_idx - l_idx    )    )

							_lowercase       :	List[Any]           =   keep_order






	def       _a(self				: Optional[int]      ,	snake_case				: Tuple      ,	snake_case				: str      ,	snake_case				: int      ,	snake_case				: Union[str, Any]    ) ->			int:
							if proj is None:
													_lowercase       :	str           =   nn.functional.linear(snake_case      ,	snake_case      ,	bias=snake_case    )
							else:
													# if CUDA_MAJOR <= 9 and CUDA_MINOR <= 1:
													_lowercase       :	Union[str, Any]           =   nn.functional.linear(snake_case      ,	proj.t().contiguous()    )
													_lowercase       :	Optional[Any]           =   nn.functional.linear(snake_case      ,	snake_case      ,	bias=snake_case    )
													# else:
													#     logit = torch.einsum('bd,de,ev->bv', (hidden, proj, weight.t()))
													#     if bias is not None:
													#         logit = logit + bias

							return logit






	def       _a(self				: str      ,	snake_case				: Dict      ,	snake_case				: int=None      ,	snake_case				: List[str]=False    ) ->			str:

							if labels is not None:
													# Shift so that tokens < n predict n
													_lowercase       :	Dict           =   hidden[..., :-1, :].contiguous()
													_lowercase       :	List[Any]           =   labels[..., 1:].contiguous()
													_lowercase       :	str           =   hidden.view(-1      ,	hidden.size(-1    )    )
													_lowercase       :	Tuple           =   labels.view(-1    )
													if hidden.size(0    ) != labels.size(0    ):
																			raise RuntimeError("Input and labels should have the same size in the batch dimension."    )
							else:
													_lowercase       :	Optional[int]           =   hidden.view(-1      ,	hidden.size(-1    )    )

							if self.n_clusters == 0:
													_lowercase       :	List[str]           =   self._compute_logit(snake_case      ,	self.out_layers[0].weight      ,	self.out_layers[0].bias      ,	self.out_projs[0]    )
													if labels is not None:
																			_lowercase       :	Tuple           =   labels != -100
																			_lowercase       :	Optional[Any]           =   torch.zeros_like(snake_case      ,	dtype=hidden.dtype      ,	device=hidden.device    )
																			_lowercase       :	Dict           =   (
																			    -nn.functional.log_softmax(snake_case      ,	dim=-1    )[mask].gather(1      ,	labels[mask].unsqueeze(1    )    ).squeeze(1    )
																			)
													else:
																			_lowercase       :	Optional[Any]           =   nn.functional.log_softmax(snake_case      ,	dim=-1    )
							else:
													# construct weights and biases
													_lowercase      ,    _lowercase       :	Any           =   [], []
													for i in range(len(self.cutoffs    )    ):
																			if self.div_val == 1:
																									_lowercase      ,    _lowercase       :	int           =   self.cutoff_ends[i], self.cutoff_ends[i + 1]
																									_lowercase       :	Optional[Any]           =   self.out_layers[0].weight[l_idx:r_idx]
																									_lowercase       :	Optional[int]           =   self.out_layers[0].bias[l_idx:r_idx]
																			else:
																									_lowercase       :	int           =   self.out_layers[i].weight
																									_lowercase       :	str           =   self.out_layers[i].bias

																			if i == 0:
																									_lowercase       :	str           =   torch.cat([weight_i, self.cluster_weight]      ,	dim=0    )
																									_lowercase       :	Optional[Any]           =   torch.cat([bias_i, self.cluster_bias]      ,	dim=0    )

																			weights.append(snake_case    )
																			biases.append(snake_case    )

													_lowercase      ,    _lowercase      ,    _lowercase       :	str           =   weights[0], biases[0], self.out_projs[0]

													_lowercase       :	Dict           =   self._compute_logit(snake_case      ,	snake_case      ,	snake_case      ,	snake_case    )
													_lowercase       :	str           =   nn.functional.log_softmax(snake_case      ,	dim=1    )

													if labels is None:
																			_lowercase       :	Optional[Any]           =   hidden.new_empty((head_logit.size(0    ), self.n_token)    )
													else:
																			_lowercase       :	List[str]           =   torch.zeros_like(snake_case      ,	dtype=hidden.dtype      ,	device=hidden.device    )

													_lowercase       :	str           =   0
													_lowercase       :	Dict           =   [0] + self.cutoffs
													for i in range(len(snake_case    ) - 1    ):
																			_lowercase      ,    _lowercase       :	Tuple           =   cutoff_values[i], cutoff_values[i + 1]

																			if labels is not None:
																									_lowercase       :	List[Any]           =   (labels >= l_idx) & (labels < r_idx)
																									_lowercase       :	int           =   mask_i.nonzero().squeeze()

																									if indices_i.numel() == 0:
																															continue

																									_lowercase       :	int           =   labels.index_select(0      ,	snake_case    ) - l_idx
																									_lowercase       :	Any           =   head_logprob.index_select(0      ,	snake_case    )
																									_lowercase       :	Dict           =   hidden.index_select(0      ,	snake_case    )
																			else:
																									_lowercase       :	Dict           =   hidden

																			if i == 0:
																									if labels is not None:
																															_lowercase       :	Dict           =   head_logprob_i.gather(1      ,	target_i[:, None]    ).squeeze(1    )
																									else:
																															_lowercase       :	Any           =   head_logprob[:, : self.cutoffs[0]]
																			else:
																									_lowercase      ,    _lowercase      ,    _lowercase       :	List[Any]           =   weights[i], biases[i], self.out_projs[i]

																									_lowercase       :	List[str]           =   self._compute_logit(snake_case      ,	snake_case      ,	snake_case      ,	snake_case    )
																									_lowercase       :	int           =   nn.functional.log_softmax(snake_case      ,	dim=1    )
																									_lowercase       :	Optional[Any]           =   self.cutoffs[0] + i - 1  # No probability for the head cluster
																									if labels is not None:
																															_lowercase       :	Dict           =   head_logprob_i[:, cluster_prob_idx] + tail_logprob_i.gather(
																															    1      ,	target_i[:, None]    ).squeeze(1    )
																									else:
																															_lowercase       :	Optional[int]           =   head_logprob[:, cluster_prob_idx, None] + tail_logprob_i
																															_lowercase       :	Optional[int]           =   logprob_i

																			if labels is not None:
																									if (hasattr(self      ,	"keep_order"    ) and self.keep_order) or keep_order:
																															out.index_copy_(0      ,	snake_case      ,	-logprob_i    )
																									else:
																															out[offset : offset + logprob_i.size(0    )].copy_(-logprob_i    )
																									offset += logprob_i.size(0    )

							return out






	def       _a(self				: Optional[int]      ,	snake_case				: int    ) ->			str:
							if self.n_clusters == 0:
													_lowercase       :	Union[str, Any]           =   self._compute_logit(snake_case      ,	self.out_layers[0].weight      ,	self.out_layers[0].bias      ,	self.out_projs[0]    )
													return nn.functional.log_softmax(snake_case      ,	dim=-1    )
							else:
													# construct weights and biases
													_lowercase      ,    _lowercase       :	List[str]           =   [], []
													for i in range(len(self.cutoffs    )    ):
																			if self.div_val == 1:
																									_lowercase      ,    _lowercase       :	Any           =   self.cutoff_ends[i], self.cutoff_ends[i + 1]
																									_lowercase       :	List[Any]           =   self.out_layers[0].weight[l_idx:r_idx]
																									_lowercase       :	Any           =   self.out_layers[0].bias[l_idx:r_idx]
																			else:
																									_lowercase       :	Any           =   self.out_layers[i].weight
																									_lowercase       :	Optional[int]           =   self.out_layers[i].bias

																			if i == 0:
																									_lowercase       :	str           =   torch.cat([weight_i, self.cluster_weight]      ,	dim=0    )
																									_lowercase       :	Tuple           =   torch.cat([bias_i, self.cluster_bias]      ,	dim=0    )

																			weights.append(snake_case    )
																			biases.append(snake_case    )

													_lowercase      ,    _lowercase      ,    _lowercase       :	Union[str, Any]           =   weights[0], biases[0], self.out_projs[0]
													_lowercase       :	Any           =   self._compute_logit(snake_case      ,	snake_case      ,	snake_case      ,	snake_case    )

													_lowercase       :	str           =   hidden.new_empty((head_logit.size(0    ), self.n_token)    )
													_lowercase       :	Optional[Any]           =   nn.functional.log_softmax(snake_case      ,	dim=1    )

													_lowercase       :	Optional[int]           =   [0] + self.cutoffs
													for i in range(len(snake_case    ) - 1    ):
																			_lowercase      ,    _lowercase       :	List[str]           =   cutoff_values[i], cutoff_values[i + 1]

																			if i == 0:
																									_lowercase       :	Optional[int]           =   head_logprob[:, : self.cutoffs[0]]
																			else:
																									_lowercase      ,    _lowercase      ,    _lowercase       :	str           =   weights[i], biases[i], self.out_projs[i]

																									_lowercase       :	Optional[Any]           =   self._compute_logit(snake_case      ,	snake_case      ,	snake_case      ,	snake_case    )
																									_lowercase       :	Union[str, Any]           =   nn.functional.log_softmax(snake_case      ,	dim=1    )

																									_lowercase       :	Optional[int]           =   head_logprob[:, -i] + tail_logprob_i
																									_lowercase       :	int           =   logprob_i

													return out






from ...configuration_utils import PretrainedConfig
from ...utils import logging


UpperCamelCase_						:     Optional[int]	       =							logging.get_logger(__name__)




class   __lowercase						(						__snake_case       ):
	_A       =   "timm_backbone"






	def __init__(self				: Any      ,	snake_case				: List[Any]=None      ,	snake_case				: int=3      ,	snake_case				: Dict=True      ,	snake_case				: Union[str, Any]=True      ,	snake_case				: Any=None      ,	**snake_case				: int      ,	) ->			Optional[int]:
							super().__init__(**snake_case    )
							_lowercase       :	Dict           =   backbone
							_lowercase       :	Optional[Any]           =   num_channels
							_lowercase       :	Union[str, Any]           =   features_only
							_lowercase       :	Tuple           =   use_pretrained_backbone
							_lowercase       :	List[str]           =   True
							_lowercase       :	Tuple           =   out_indices if out_indices is not None else (-1,)




