





import torch
from torch import nn





class 				a			(      nn.Module			):






				def __init__(  self			, _lowerCAmelCase			, _lowerCAmelCase			, _lowerCAmelCase			, _lowerCAmelCase			, _lowerCAmelCase=1			, _lowerCAmelCase=False					):

										"""simple docstring"""




										super().__init__()

										__SCREAMING_SNAKE_CASE:				Union[str, Any]           =   n_token
										__SCREAMING_SNAKE_CASE:				Union[str, Any]           =   d_embed
										__SCREAMING_SNAKE_CASE:				Optional[Any]           =   d_proj

										__SCREAMING_SNAKE_CASE:				List[Any]           =   cutoffs + [n_token]
										__SCREAMING_SNAKE_CASE:				str           =   [0] + self.cutoffs
										__SCREAMING_SNAKE_CASE:				str           =   div_val

										__SCREAMING_SNAKE_CASE:				Tuple           =   self.cutoffs[0]
										__SCREAMING_SNAKE_CASE:				str           =   len(self.cutoffs					) - 1
										__SCREAMING_SNAKE_CASE:				int           =   self.shortlist_size + self.n_clusters

										if self.n_clusters > 0:
																__SCREAMING_SNAKE_CASE:				Union[str, Any]           =   nn.Parameter(torch.zeros(self.n_clusters			, self.d_embed					)					)
																__SCREAMING_SNAKE_CASE:				List[str]           =   nn.Parameter(torch.zeros(self.n_clusters					)					)

										__SCREAMING_SNAKE_CASE:				int           =   nn.ModuleList()
										__SCREAMING_SNAKE_CASE:				Optional[Any]           =   nn.ParameterList()

										if div_val == 1:
																for i in range(len(self.cutoffs					)					):
																						if d_proj != d_embed:
																												self.out_projs.append(nn.Parameter(torch.FloatTensor(_lowerCAmelCase			, _lowerCAmelCase					)					)					)
																						else:
																												self.out_projs.append(_lowerCAmelCase					)

																self.out_layers.append(nn.Linear(_lowerCAmelCase			, _lowerCAmelCase					)					)
										else:
																for i in range(len(self.cutoffs					)					):
																						__SCREAMING_SNAKE_CASE       ,__SCREAMING_SNAKE_CASE:				str           =   self.cutoff_ends[i], self.cutoff_ends[i + 1]
																						__SCREAMING_SNAKE_CASE:				str           =   d_embed // (div_val**i)

																						self.out_projs.append(nn.Parameter(torch.FloatTensor(_lowerCAmelCase			, _lowerCAmelCase					)					)					)

																						self.out_layers.append(nn.Linear(_lowerCAmelCase			, r_idx - l_idx					)					)

										__SCREAMING_SNAKE_CASE:				Optional[Any]           =   keep_order






				def 			snake_case_    (  self			, _lowerCAmelCase			, _lowerCAmelCase			, _lowerCAmelCase			, _lowerCAmelCase					):

										"""simple docstring"""




										if proj is None:
																__SCREAMING_SNAKE_CASE:				Union[str, Any]           =   nn.functional.linear(_lowerCAmelCase			, _lowerCAmelCase			, bias=_lowerCAmelCase					)
										else:
																# if CUDA_MAJOR <= 9 and CUDA_MINOR <= 1:
																__SCREAMING_SNAKE_CASE:				Tuple           =   nn.functional.linear(_lowerCAmelCase			, proj.t().contiguous()					)
																__SCREAMING_SNAKE_CASE:				Optional[int]           =   nn.functional.linear(_lowerCAmelCase			, _lowerCAmelCase			, bias=_lowerCAmelCase					)
																# else:
																#     logit = torch.einsum('bd,de,ev->bv', (hidden, proj, weight.t()))
																#     if bias is not None:
																#         logit = logit + bias

										return logit






				def 			snake_case_    (  self			, _lowerCAmelCase			, _lowerCAmelCase=None			, _lowerCAmelCase=False					):

										"""simple docstring"""




										if labels is not None:
																# Shift so that tokens < n predict n
																__SCREAMING_SNAKE_CASE:				Optional[int]           =   hidden[..., :-1, :].contiguous()
																__SCREAMING_SNAKE_CASE:				Union[str, Any]           =   labels[..., 1:].contiguous()
																__SCREAMING_SNAKE_CASE:				Optional[Any]           =   hidden.view(-1			, hidden.size(-1					)					)
																__SCREAMING_SNAKE_CASE:				str           =   labels.view(-1					)
																if hidden.size(0					) != labels.size(0					):
																						raise RuntimeError('''Input and labels should have the same size in the batch dimension.'''					)
										else:
																__SCREAMING_SNAKE_CASE:				int           =   hidden.view(-1			, hidden.size(-1					)					)

										if self.n_clusters == 0:
																__SCREAMING_SNAKE_CASE:				List[str]           =   self._compute_logit(_lowerCAmelCase			, self.out_layers[0].weight			, self.out_layers[0].bias			, self.out_projs[0]					)
																if labels is not None:
																						__SCREAMING_SNAKE_CASE:				List[Any]           =   labels != -100
																						__SCREAMING_SNAKE_CASE:				str           =   torch.zeros_like(_lowerCAmelCase			, dtype=hidden.dtype			, device=hidden.device					)
																						__SCREAMING_SNAKE_CASE:				Dict           =   (
																						    -nn.functional.log_softmax(_lowerCAmelCase			, dim=-1					)[mask].gather(1			, labels[mask].unsqueeze(1					)					).squeeze(1					)
																						)
																else:
																						__SCREAMING_SNAKE_CASE:				Optional[int]           =   nn.functional.log_softmax(_lowerCAmelCase			, dim=-1					)
										else:
																# construct weights and biases
																__SCREAMING_SNAKE_CASE       ,__SCREAMING_SNAKE_CASE:				str           =   [], []
																for i in range(len(self.cutoffs					)					):
																						if self.div_val == 1:
																												__SCREAMING_SNAKE_CASE       ,__SCREAMING_SNAKE_CASE:				List[str]           =   self.cutoff_ends[i], self.cutoff_ends[i + 1]
																												__SCREAMING_SNAKE_CASE:				Optional[Any]           =   self.out_layers[0].weight[l_idx:r_idx]
																												__SCREAMING_SNAKE_CASE:				List[str]           =   self.out_layers[0].bias[l_idx:r_idx]
																						else:
																												__SCREAMING_SNAKE_CASE:				Dict           =   self.out_layers[i].weight
																												__SCREAMING_SNAKE_CASE:				Any           =   self.out_layers[i].bias

																						if i == 0:
																												__SCREAMING_SNAKE_CASE:				Optional[int]           =   torch.cat([weight_i, self.cluster_weight]			, dim=0					)
																												__SCREAMING_SNAKE_CASE:				str           =   torch.cat([bias_i, self.cluster_bias]			, dim=0					)

																						weights.append(_lowerCAmelCase					)
																						biases.append(_lowerCAmelCase					)

																__SCREAMING_SNAKE_CASE       ,__SCREAMING_SNAKE_CASE       ,__SCREAMING_SNAKE_CASE:				str           =   weights[0], biases[0], self.out_projs[0]

																__SCREAMING_SNAKE_CASE:				str           =   self._compute_logit(_lowerCAmelCase			, _lowerCAmelCase			, _lowerCAmelCase			, _lowerCAmelCase					)
																__SCREAMING_SNAKE_CASE:				Any           =   nn.functional.log_softmax(_lowerCAmelCase			, dim=1					)

																if labels is None:
																						__SCREAMING_SNAKE_CASE:				int           =   hidden.new_empty((head_logit.size(0					), self.n_token)					)
																else:
																						__SCREAMING_SNAKE_CASE:				str           =   torch.zeros_like(_lowerCAmelCase			, dtype=hidden.dtype			, device=hidden.device					)

																__SCREAMING_SNAKE_CASE:				Tuple           =   0
																__SCREAMING_SNAKE_CASE:				List[str]           =   [0] + self.cutoffs
																for i in range(len(_lowerCAmelCase					) - 1					):
																						__SCREAMING_SNAKE_CASE       ,__SCREAMING_SNAKE_CASE:				List[str]           =   cutoff_values[i], cutoff_values[i + 1]

																						if labels is not None:
																												__SCREAMING_SNAKE_CASE:				List[Any]           =   (labels >= l_idx) & (labels < r_idx)
																												__SCREAMING_SNAKE_CASE:				List[str]           =   mask_i.nonzero().squeeze()

																												if indices_i.numel() == 0:
																																		continue

																												__SCREAMING_SNAKE_CASE:				Tuple           =   labels.index_select(0			, _lowerCAmelCase					) - l_idx
																												__SCREAMING_SNAKE_CASE:				str           =   head_logprob.index_select(0			, _lowerCAmelCase					)
																												__SCREAMING_SNAKE_CASE:				Tuple           =   hidden.index_select(0			, _lowerCAmelCase					)
																						else:
																												__SCREAMING_SNAKE_CASE:				Optional[int]           =   hidden

																						if i == 0:
																												if labels is not None:
																																		__SCREAMING_SNAKE_CASE:				Dict           =   head_logprob_i.gather(1			, target_i[:, None]					).squeeze(1					)
																												else:
																																		__SCREAMING_SNAKE_CASE:				Optional[int]           =   head_logprob[:, : self.cutoffs[0]]
																						else:
																												__SCREAMING_SNAKE_CASE       ,__SCREAMING_SNAKE_CASE       ,__SCREAMING_SNAKE_CASE:				Tuple           =   weights[i], biases[i], self.out_projs[i]

																												__SCREAMING_SNAKE_CASE:				Optional[int]           =   self._compute_logit(_lowerCAmelCase			, _lowerCAmelCase			, _lowerCAmelCase			, _lowerCAmelCase					)
																												__SCREAMING_SNAKE_CASE:				Union[str, Any]           =   nn.functional.log_softmax(_lowerCAmelCase			, dim=1					)
																												__SCREAMING_SNAKE_CASE:				Union[str, Any]           =   self.cutoffs[0] + i - 1  # No probability for the head cluster
																												if labels is not None:
																																		__SCREAMING_SNAKE_CASE:				Union[str, Any]           =   head_logprob_i[:, cluster_prob_idx] + tail_logprob_i.gather(
																																		    1			, target_i[:, None]					).squeeze(1					)
																												else:
																																		__SCREAMING_SNAKE_CASE:				Union[str, Any]           =   head_logprob[:, cluster_prob_idx, None] + tail_logprob_i
																																		__SCREAMING_SNAKE_CASE:				str           =   logprob_i

																						if labels is not None:
																												if (hasattr(self			, '''keep_order'''					) and self.keep_order) or keep_order:
																																		out.index_copy_(0			, _lowerCAmelCase			, -logprob_i					)
																												else:
																																		out[offset : offset + logprob_i.size(0					)].copy_(-logprob_i					)
																												offset += logprob_i.size(0					)

										return out






				def 			snake_case_    (  self			, _lowerCAmelCase					):

										"""simple docstring"""




										if self.n_clusters == 0:
																__SCREAMING_SNAKE_CASE:				Tuple           =   self._compute_logit(_lowerCAmelCase			, self.out_layers[0].weight			, self.out_layers[0].bias			, self.out_projs[0]					)
																return nn.functional.log_softmax(_lowerCAmelCase			, dim=-1					)
										else:
																# construct weights and biases
																__SCREAMING_SNAKE_CASE       ,__SCREAMING_SNAKE_CASE:				List[str]           =   [], []
																for i in range(len(self.cutoffs					)					):
																						if self.div_val == 1:
																												__SCREAMING_SNAKE_CASE       ,__SCREAMING_SNAKE_CASE:				Dict           =   self.cutoff_ends[i], self.cutoff_ends[i + 1]
																												__SCREAMING_SNAKE_CASE:				List[Any]           =   self.out_layers[0].weight[l_idx:r_idx]
																												__SCREAMING_SNAKE_CASE:				str           =   self.out_layers[0].bias[l_idx:r_idx]
																						else:
																												__SCREAMING_SNAKE_CASE:				List[Any]           =   self.out_layers[i].weight
																												__SCREAMING_SNAKE_CASE:				str           =   self.out_layers[i].bias

																						if i == 0:
																												__SCREAMING_SNAKE_CASE:				Optional[int]           =   torch.cat([weight_i, self.cluster_weight]			, dim=0					)
																												__SCREAMING_SNAKE_CASE:				List[Any]           =   torch.cat([bias_i, self.cluster_bias]			, dim=0					)

																						weights.append(_lowerCAmelCase					)
																						biases.append(_lowerCAmelCase					)

																__SCREAMING_SNAKE_CASE       ,__SCREAMING_SNAKE_CASE       ,__SCREAMING_SNAKE_CASE:				Optional[Any]           =   weights[0], biases[0], self.out_projs[0]
																__SCREAMING_SNAKE_CASE:				Tuple           =   self._compute_logit(_lowerCAmelCase			, _lowerCAmelCase			, _lowerCAmelCase			, _lowerCAmelCase					)

																__SCREAMING_SNAKE_CASE:				Optional[Any]           =   hidden.new_empty((head_logit.size(0					), self.n_token)					)
																__SCREAMING_SNAKE_CASE:				Optional[Any]           =   nn.functional.log_softmax(_lowerCAmelCase			, dim=1					)

																__SCREAMING_SNAKE_CASE:				Tuple           =   [0] + self.cutoffs
																for i in range(len(_lowerCAmelCase					) - 1					):
																						__SCREAMING_SNAKE_CASE       ,__SCREAMING_SNAKE_CASE:				Union[str, Any]           =   cutoff_values[i], cutoff_values[i + 1]

																						if i == 0:
																												__SCREAMING_SNAKE_CASE:				List[str]           =   head_logprob[:, : self.cutoffs[0]]
																						else:
																												__SCREAMING_SNAKE_CASE       ,__SCREAMING_SNAKE_CASE       ,__SCREAMING_SNAKE_CASE:				Dict           =   weights[i], biases[i], self.out_projs[i]

																												__SCREAMING_SNAKE_CASE:				Any           =   self._compute_logit(_lowerCAmelCase			, _lowerCAmelCase			, _lowerCAmelCase			, _lowerCAmelCase					)
																												__SCREAMING_SNAKE_CASE:				Optional[Any]           =   nn.functional.log_softmax(_lowerCAmelCase			, dim=1					)

																												__SCREAMING_SNAKE_CASE:				str           =   head_logprob[:, -i] + tail_logprob_i
																												__SCREAMING_SNAKE_CASE:				List[Any]           =   logprob_i

																return out










import unittest
from dataclasses import dataclass

import pytest

from accelerate.commands.config.config_args import SageMakerConfig
from accelerate.utils import ComputeEnvironment
from accelerate.utils.launch import _convert_nargs_to_dict





@dataclass
class 				a			(      __lowercase			):
				SCREAMING_SNAKE_CASE__     :   Any		      =     ComputeEnvironment.AMAZON_SAGEMAKER
				SCREAMING_SNAKE_CASE__     :   Optional[int]		      =     True
				SCREAMING_SNAKE_CASE__     :   int		      =     '''ml.p3.2xlarge'''
				SCREAMING_SNAKE_CASE__     :   Union[str, Any]		      =     '''accelerate_sagemaker_execution_role'''
				SCREAMING_SNAKE_CASE__     :   List[Any]		      =     '''hf-sm'''
				SCREAMING_SNAKE_CASE__     :   Any		      =     '''us-east-1'''
				SCREAMING_SNAKE_CASE__     :   Any		      =     1
				SCREAMING_SNAKE_CASE__     :   Dict		      =     '''accelerate-sagemaker-1'''
				SCREAMING_SNAKE_CASE__     :   Dict		      =     '''1.6'''
				SCREAMING_SNAKE_CASE__     :   int		      =     '''4.4'''
				SCREAMING_SNAKE_CASE__     :   int		      =     '''train.py'''
				SCREAMING_SNAKE_CASE__     :   Tuple		      =     [
				    '''--model_name_or_path''',
				    '''bert''',
				    '''--do_train''',
				    '''False''',
				    '''--epochs''',
				    '''3''',
				    '''--learning_rate''',
				    '''5e-5''',
				    '''--max_steps''',
				    '''50.5''',
				]
				SCREAMING_SNAKE_CASE__     :   List[str]		      =     [
				    '''--model_name_or_path''',
				    '''bert''',
				    '''--do_train''',
				    '''--do_test''',
				    '''False''',
				    '''--do_predict''',
				    '''--epochs''',
				    '''3''',
				    '''--learning_rate''',
				    '''5e-5''',
				    '''--max_steps''',
				    '''50.5''',
				]





class 				a			(      unittest.TestCase			):






				def 			snake_case_    (  self					):

										"""simple docstring"""




										__SCREAMING_SNAKE_CASE:				Optional[Any]           =   _convert_nargs_to_dict(MockLaunchConfig.success_training_script_args					)
										assert isinstance(converted_args['''model_name_or_path''']			, _lowerCAmelCase					)
										assert isinstance(converted_args['''do_train''']			, _lowerCAmelCase					)
										assert isinstance(converted_args['''epochs''']			, _lowerCAmelCase					)
										assert isinstance(converted_args['''learning_rate''']			, _lowerCAmelCase					)
										assert isinstance(converted_args['''max_steps''']			, _lowerCAmelCase					)

										with pytest.raises(_lowerCAmelCase					):
																_convert_nargs_to_dict(MockLaunchConfig.fail_training_script_args					)



