



'''simple docstring'''






import torch
from torch import nn

class 							a__(      nn.Module			):







				def __init__(   self				:						Optional[int]		,    __snake_case				:						Dict		,    __snake_case				:						Tuple		,    __snake_case				:						Union[str, Any]		,    __snake_case				:						List[Any]		,    __snake_case				:						Tuple=1		,    __snake_case				:						str=False			):
						super().__init__()

						a		:      int       =  n_token
						a		:      List[str]       =  d_embed
						a		:      Union[str, Any]       =  d_proj

						a		:      str       =  cutoffs + [n_token]
						a		:      Optional[int]       =  [0] + self.cutoffs
						a		:      int       =  div_val

						a		:      Tuple       =  self.cutoffs[0]
						a		:      str       =  len(self.cutoffs			) - 1
						a		:      List[Any]       =  self.shortlist_size + self.n_clusters

						if self.n_clusters > 0:
								a		:      Tuple       =  nn.Parameter(torch.zeros(self.n_clusters		,    self.d_embed			)			)
								a		:      List[Any]       =  nn.Parameter(torch.zeros(self.n_clusters			)			)

						a		:      Any       =  nn.ModuleList()
						a		:      Tuple       =  nn.ParameterList()

						if div_val == 1:
								for i in range(len(self.cutoffs			)			):
										if d_proj != d_embed:
												self.out_projs.append(nn.Parameter(torch.FloatTensor(__snake_case		,    __snake_case			)			)			)
										else:
												self.out_projs.append(__snake_case			)

								self.out_layers.append(nn.Linear(__snake_case		,    __snake_case			)			)
						else:
								for i in range(len(self.cutoffs			)			):
										a		:      Optional[int]       =  self.cutoff_ends[i], self.cutoff_ends[i + 1]
										a		:      Union[str, Any]       =  d_embed // (div_val**i)

										self.out_projs.append(nn.Parameter(torch.FloatTensor(__snake_case		,    __snake_case			)			)			)

										self.out_layers.append(nn.Linear(__snake_case		,    r_idx - l_idx			)			)

						a		:      Optional[int]       =  keep_order







				def     lowercase_ (   self				:						Optional[Any]		,    __snake_case				:						Dict		,    __snake_case				:						int		,    __snake_case				:						Tuple		,    __snake_case				:						Optional[Any]			):
						if proj is None:
								a		:      List[str]       =  nn.functional.linear(__snake_case		,    __snake_case		,    bias=__snake_case			)
						else:
								# if CUDA_MAJOR <= 9 and CUDA_MINOR <= 1:
								a		:      int       =  nn.functional.linear(__snake_case		,    proj.t().contiguous()			)
								a		:      Union[str, Any]       =  nn.functional.linear(__snake_case		,    __snake_case		,    bias=__snake_case			)
								# else:
								#     logit = torch.einsum('bd,de,ev->bv', (hidden, proj, weight.t()))
								#     if bias is not None:
								#         logit = logit + bias

						return logit







				def     lowercase_ (   self				:						Optional[int]		,    __snake_case				:						Optional[Any]		,    __snake_case				:						List[Any]=None		,    __snake_case				:						Union[str, Any]=False			):

						if labels is not None:
								# Shift so that tokens < n predict n
								a		:      List[str]       =  hidden[..., :-1, :].contiguous()
								a		:      Union[str, Any]       =  labels[..., 1:].contiguous()
								a		:      Dict       =  hidden.view(-1		,    hidden.size(-1			)			)
								a		:      Tuple       =  labels.view(-1			)
								if hidden.size(0			) != labels.size(0			):
										raise RuntimeError('Input and labels should have the same size in the batch dimension.'			)
						else:
								a		:      List[Any]       =  hidden.view(-1		,    hidden.size(-1			)			)

						if self.n_clusters == 0:
								a		:      Tuple       =  self._compute_logit(__snake_case		,    self.out_layers[0].weight		,    self.out_layers[0].bias		,    self.out_projs[0]			)
								if labels is not None:
										a		:      Tuple       =  labels != -1_00
										a		:      Optional[int]       =  torch.zeros_like(__snake_case		,    dtype=hidden.dtype		,    device=hidden.device			)
										a		:      List[Any]       =  (
										    -nn.functional.log_softmax(__snake_case		,    dim=-1			)[mask].gather(1		,    labels[mask].unsqueeze(1			)			).squeeze(1			)
										)
								else:
										a		:      Dict       =  nn.functional.log_softmax(__snake_case		,    dim=-1			)
						else:
								# construct weights and biases
								a		:      Dict       =  [], []
								for i in range(len(self.cutoffs			)			):
										if self.div_val == 1:
												a		:      Union[str, Any]       =  self.cutoff_ends[i], self.cutoff_ends[i + 1]
												a		:      Dict       =  self.out_layers[0].weight[l_idx:r_idx]
												a		:      List[str]       =  self.out_layers[0].bias[l_idx:r_idx]
										else:
												a		:      Optional[Any]       =  self.out_layers[i].weight
												a		:      int       =  self.out_layers[i].bias

										if i == 0:
												a		:      Tuple       =  torch.cat([weight_i, self.cluster_weight]		,    dim=0			)
												a		:      Union[str, Any]       =  torch.cat([bias_i, self.cluster_bias]		,    dim=0			)

										weights.append(__snake_case			)
										biases.append(__snake_case			)

								a		:      Dict       =  weights[0], biases[0], self.out_projs[0]

								a		:      Optional[int]       =  self._compute_logit(__snake_case		,    __snake_case		,    __snake_case		,    __snake_case			)
								a		:      int       =  nn.functional.log_softmax(__snake_case		,    dim=1			)

								if labels is None:
										a		:      Dict       =  hidden.new_empty((head_logit.size(0			), self.n_token)			)
								else:
										a		:      Dict       =  torch.zeros_like(__snake_case		,    dtype=hidden.dtype		,    device=hidden.device			)

								a		:      List[Any]       =  0
								a		:      str       =  [0] + self.cutoffs
								for i in range(len(__snake_case			) - 1			):
										a		:      List[str]       =  cutoff_values[i], cutoff_values[i + 1]

										if labels is not None:
												a		:      Union[str, Any]       =  (labels >= l_idx) & (labels < r_idx)
												a		:      Union[str, Any]       =  mask_i.nonzero().squeeze()

												if indices_i.numel() == 0:
														continue

												a		:      int       =  labels.index_select(0		,    __snake_case			) - l_idx
												a		:      Any       =  head_logprob.index_select(0		,    __snake_case			)
												a		:      List[str]       =  hidden.index_select(0		,    __snake_case			)
										else:
												a		:      Tuple       =  hidden

										if i == 0:
												if labels is not None:
														a		:      Optional[Any]       =  head_logprob_i.gather(1		,    target_i[:, None]			).squeeze(1			)
												else:
														a		:      int       =  head_logprob[:, : self.cutoffs[0]]
										else:
												a		:      List[str]       =  weights[i], biases[i], self.out_projs[i]

												a		:      Tuple       =  self._compute_logit(__snake_case		,    __snake_case		,    __snake_case		,    __snake_case			)
												a		:      int       =  nn.functional.log_softmax(__snake_case		,    dim=1			)
												a		:      List[str]       =  self.cutoffs[0] + i - 1  # No probability for the head cluster
												if labels is not None:
														a		:      str       =  head_logprob_i[:, cluster_prob_idx] + tail_logprob_i.gather(
														    1		,    target_i[:, None]			).squeeze(1			)
												else:
														a		:      Optional[Any]       =  head_logprob[:, cluster_prob_idx, None] + tail_logprob_i
														a		:      Optional[Any]       =  logprob_i

										if labels is not None:
												if (hasattr(self		,    'keep_order'			) and self.keep_order) or keep_order:
														out.index_copy_(0		,    __snake_case		,    -logprob_i			)
												else:
														out[offset : offset + logprob_i.size(0			)].copy_(-logprob_i			)
												offset += logprob_i.size(0			)

						return out







				def     lowercase_ (   self				:						Optional[Any]		,    __snake_case				:						Dict			):
						if self.n_clusters == 0:
								a		:      List[str]       =  self._compute_logit(__snake_case		,    self.out_layers[0].weight		,    self.out_layers[0].bias		,    self.out_projs[0]			)
								return nn.functional.log_softmax(__snake_case		,    dim=-1			)
						else:
								# construct weights and biases
								a		:      Dict       =  [], []
								for i in range(len(self.cutoffs			)			):
										if self.div_val == 1:
												a		:      Optional[Any]       =  self.cutoff_ends[i], self.cutoff_ends[i + 1]
												a		:      Any       =  self.out_layers[0].weight[l_idx:r_idx]
												a		:      List[Any]       =  self.out_layers[0].bias[l_idx:r_idx]
										else:
												a		:      str       =  self.out_layers[i].weight
												a		:      List[str]       =  self.out_layers[i].bias

										if i == 0:
												a		:      Union[str, Any]       =  torch.cat([weight_i, self.cluster_weight]		,    dim=0			)
												a		:      Any       =  torch.cat([bias_i, self.cluster_bias]		,    dim=0			)

										weights.append(__snake_case			)
										biases.append(__snake_case			)

								a		:      int       =  weights[0], biases[0], self.out_projs[0]
								a		:      Tuple       =  self._compute_logit(__snake_case		,    __snake_case		,    __snake_case		,    __snake_case			)

								a		:      Tuple       =  hidden.new_empty((head_logit.size(0			), self.n_token)			)
								a		:      Optional[Any]       =  nn.functional.log_softmax(__snake_case		,    dim=1			)

								a		:      List[str]       =  [0] + self.cutoffs
								for i in range(len(__snake_case			) - 1			):
										a		:      Tuple       =  cutoff_values[i], cutoff_values[i + 1]

										if i == 0:
												a		:      Any       =  head_logprob[:, : self.cutoffs[0]]
										else:
												a		:      Optional[Any]       =  weights[i], biases[i], self.out_projs[i]

												a		:      Dict       =  self._compute_logit(__snake_case		,    __snake_case		,    __snake_case		,    __snake_case			)
												a		:      Optional[Any]       =  nn.functional.log_softmax(__snake_case		,    dim=1			)

												a		:      int       =  head_logprob[:, -i] + tail_logprob_i
												a		:      List[str]       =  logprob_i

								return out




'''simple docstring'''






from __future__ import annotations

from sys import maxsize
from typing import Generic, TypeVar

lowerCAmelCase:      List[str]		 =		TypeVar('T')
def    lowerCamelCase__ (		_A       ):
  return (position - 1) // 2
def    lowerCamelCase__ (		_A       ):
  return (2 * position) + 1
def    lowerCamelCase__ (		_A       ):
  return (2 * position) + 2

class 							a__(      Generic[T]			):







    def __init__(   self				:						Optional[Any]			):
      a		:      list[tuple[T, int]]       =  []
      a		:      dict[T, int]       =  {}
      a		:      int       =  0







    def __len__(   self				:						Union[str, Any]			):
      return self.elements







    def __repr__(   self				:						Any			):
      return str(self.heap			)







    def     lowercase_ (   self				:						Optional[int]			):
      # Check if the priority queue is empty
      return self.elements == 0







    def     lowercase_ (   self				:						Union[str, Any]		,    __snake_case				:						T		,    __snake_case				:						int			):
      # Add an element with given priority to the queue
      self.heap.append((elem, weight)			)
      a		:      int       =  self.elements
      self.elements += 1
      self._bubble_up(__snake_case			)







    def     lowercase_ (   self				:						Union[str, Any]			):
      # Remove and return the element with lowest weight (highest priority)
      if self.elements > 1:
        self._swap_nodes(0		,    self.elements - 1			)
      a     ,			a		:      str       =  self.heap.pop()
      del self.position_map[elem]
      self.elements -= 1
      if self.elements > 0:
        a     ,			a		:      List[Any]       =  self.heap[0]
        self._bubble_down(__snake_case			)
      return elem







    def     lowercase_ (   self				:						List[str]		,    __snake_case				:						T		,    __snake_case				:						int			):
      # Update the weight of the given key
      a		:      Any       =  self.position_map[elem]
      a		:      Union[str, Any]       =  (elem, weight)
      if position > 0:
        a		:      Optional[Any]       =  get_parent_position(__snake_case			)
        a     ,			a		:      Union[str, Any]       =  self.heap[parent_position]
        if parent_weight > weight:
          self._bubble_up(__snake_case			)
        else:
          self._bubble_down(__snake_case			)
      else:
        self._bubble_down(__snake_case			)







    def     lowercase_ (   self				:						int		,    __snake_case				:						T			):
      # Place a node at the proper position (upward movement) [to be used internally
      # only]
      a		:      Union[str, Any]       =  self.position_map[elem]
      if curr_pos == 0:
        return None
      a		:      Union[str, Any]       =  get_parent_position(__snake_case			)
      a     ,			a		:      Tuple       =  self.heap[curr_pos]
      a     ,			a		:      List[Any]       =  self.heap[parent_position]
      if parent_weight > weight:
        self._swap_nodes(__snake_case		,    __snake_case			)
        return self._bubble_up(__snake_case			)
      return None







    def     lowercase_ (   self				:						Tuple		,    __snake_case				:						T			):
      # Place a node at the proper position (downward movement) [to be used
      # internally only]
      a		:      str       =  self.position_map[elem]
      a     ,			a		:      List[Any]       =  self.heap[curr_pos]
      a		:      int       =  get_child_left_position(__snake_case			)
      a		:      Dict       =  get_child_right_position(__snake_case			)
      if child_left_position < self.elements and child_right_position < self.elements:
        a     ,			a		:      int       =  self.heap[child_left_position]
        a     ,			a		:      Optional[Any]       =  self.heap[child_right_position]
        if child_right_weight < child_left_weight and child_right_weight < weight:
          self._swap_nodes(__snake_case		,    __snake_case			)
          return self._bubble_down(__snake_case			)
      if child_left_position < self.elements:
        a     ,			a		:      Dict       =  self.heap[child_left_position]
        if child_left_weight < weight:
          self._swap_nodes(__snake_case		,    __snake_case			)
          return self._bubble_down(__snake_case			)
      else:
        return None
      if child_right_position < self.elements:
        a     ,			a		:      List[Any]       =  self.heap[child_right_position]
        if child_right_weight < weight:
          self._swap_nodes(__snake_case		,    __snake_case			)
          return self._bubble_down(__snake_case			)
      return None







    def     lowercase_ (   self				:						List[Any]		,    __snake_case				:						int		,    __snake_case				:						int			):
      # Swap the nodes at the given positions
      a		:      Optional[int]       =  self.heap[nodea_pos][0]
      a		:      int       =  self.heap[nodea_pos][0]
      a     ,			a		:      List[str]       =  (
          self.heap[nodea_pos],
          self.heap[nodea_pos],
      )
      a		:      Tuple       =  nodea_pos
      a		:      Any       =  nodea_pos

class 							a__(      Generic[T]			):







    def __init__(   self				:						Optional[int]			):
      a		:      dict[T, dict[T, int]]       =  {}
      a		:      int       =  0







    def __repr__(   self				:						List[str]			):
      return str(self.connections			)







    def __len__(   self				:						str			):
      return self.nodes







    def     lowercase_ (   self				:						int		,    __snake_case				:						T			):
      # Add a node in the graph if it is not in the graph
      if node not in self.connections:
        a		:      Optional[int]       =  {}
        self.nodes += 1







    def     lowercase_ (   self				:						Dict		,    __snake_case				:						T		,    __snake_case				:						T		,    __snake_case				:						int			):
      # Add an edge between 2 nodes in the graph
      self.add_node(__snake_case			)
      self.add_node(__snake_case			)
      a		:      Any       =  weight
      a		:      Dict       =  weight
def    lowerCamelCase__ (		_A				,    ):
  a		:      dict[T, int]       =  {node: maxsize for node in graph.connections}
  a		:      dict[T, T | None]       =  {node: None for node in graph.connections}

  a		:      MinPriorityQueue[T]       =  MinPriorityQueue()
  for node, weight in dist.items():
    priority_queue.push(_A				,    _A       )

  if priority_queue.is_empty():
    return dist, parent

  # initialization
  a		:      int       =  priority_queue.extract_min()
  a		:      Dict       =  0
  for neighbour in graph.connections[node]:
    if dist[neighbour] > dist[node] + graph.connections[node][neighbour]:
      a		:      Tuple       =  dist[node] + graph.connections[node][neighbour]
      priority_queue.update_key(_A				,    dist[neighbour]       )
      a		:      Dict       =  node

    # running prim's algorithm
  while not priority_queue.is_empty():
    a		:      List[str]       =  priority_queue.extract_min()
    for neighbour in graph.connections[node]:
      if dist[neighbour] > dist[node] + graph.connections[node][neighbour]:
        a		:      int       =  dist[node] + graph.connections[node][neighbour]
        priority_queue.update_key(_A				,    dist[neighbour]       )
        a		:      Optional[int]       =  node
  return dist, parent