



import torch
from torch import nn

class        snake_case__    (nn.Module		):







						"""simple docstring"""




						def __init__(       self ,       __lowercase ,       __lowercase ,       __lowercase ,       __lowercase ,       __lowercase=1 ,       __lowercase=False  ) ->				Union[str, Any]:
									"""simple docstring"""




									super().__init__()

									a__       :						int							=  n_token
									a__       :						Union[str, Any]							=  d_embed
									a__       :						Optional[int]							=  d_proj

									a__       :						Union[str, Any]							=  cutoffs + [n_token]
									a__       :						Optional[int]							=  [0] + self.cutoffs
									a__       :						Optional[Any]							=  div_val

									a__       :						Optional[Any]							=  self.cutoffs[0]
									a__       :						Optional[Any]							=  len(self.cutoffs  ) - 1
									a__       :						Tuple							=  self.shortlist_size + self.n_clusters

									if self.n_clusters > 0:
												a__       :						Any							=  nn.Parameter(torch.zeros(self.n_clusters ,       self.d_embed  )  )
												a__       :						int							=  nn.Parameter(torch.zeros(self.n_clusters  )  )

									a__       :						List[Any]							=  nn.ModuleList()
									a__       :						Optional[Any]							=  nn.ParameterList()

									if div_val == 1:
												for i in range(len(self.cutoffs  )  ):
															if d_proj != d_embed:
																		self.out_projs.append(nn.Parameter(torch.FloatTensor(__lowercase ,       __lowercase  )  )  )
															else:
																		self.out_projs.append(__lowercase  )

												self.out_layers.append(nn.Linear(__lowercase ,       __lowercase  )  )
									else:
												for i in range(len(self.cutoffs  )  ):
															a__       :						str							=  self.cutoff_ends[i], self.cutoff_ends[i + 1]
															a__       :						Union[str, Any]							=  d_embed // (div_val**i)

															self.out_projs.append(nn.Parameter(torch.FloatTensor(__lowercase ,       __lowercase  )  )  )

															self.out_layers.append(nn.Linear(__lowercase ,       r_idx - l_idx  )  )

									a__       :						List[Any]							=  keep_order

						def 							SCREAMING_SNAKE_CASE__(       self ,       __lowercase ,       __lowercase ,       __lowercase ,       __lowercase  ) ->				List[str]:
									"""simple docstring"""




									if proj is None:
												a__       :						List[str]							=  nn.functional.linear(__lowercase ,       __lowercase ,       bias=__lowercase  )
									else:
												# if CUDA_MAJOR <= 9 and CUDA_MINOR <= 1:
												a__       :						Optional[int]							=  nn.functional.linear(__lowercase ,       proj.t().contiguous()  )
												a__       :						int							=  nn.functional.linear(__lowercase ,       __lowercase ,       bias=__lowercase  )
												# else:
												#     logit = torch.einsum('bd,de,ev->bv', (hidden, proj, weight.t()))
												#     if bias is not None:
												#         logit = logit + bias

									return logit

						def 							SCREAMING_SNAKE_CASE__(       self ,       __lowercase ,       __lowercase=None ,       __lowercase=False  ) ->				Optional[int]:
									"""simple docstring"""




									if labels is not None:
												# Shift so that tokens < n predict n
												a__       :						Optional[Any]							=  hidden[..., :-1, :].contiguous()
												a__       :						Dict							=  labels[..., 1:].contiguous()
												a__       :						Tuple							=  hidden.view(-1 ,       hidden.size(-1  )  )
												a__       :						Dict							=  labels.view(-1  )
												if hidden.size(0  ) != labels.size(0  ):
															raise RuntimeError("""Input and labels should have the same size in the batch dimension."""  )
									else:
												a__       :						Dict							=  hidden.view(-1 ,       hidden.size(-1  )  )

									if self.n_clusters == 0:
												a__       :						Optional[int]							=  self._compute_logit(__lowercase ,       self.out_layers[0].weight ,       self.out_layers[0].bias ,       self.out_projs[0]  )
												if labels is not None:
															a__       :						List[Any]							=  labels != -1_0_0
															a__       :						Any							=  torch.zeros_like(__lowercase ,       dtype=hidden.dtype ,       device=hidden.device  )
															a__       :						List[Any]							=  (
															    -nn.functional.log_softmax(__lowercase ,       dim=-1  )[mask].gather(1 ,       labels[mask].unsqueeze(1  )  ).squeeze(1  )
															)
												else:
															a__       :						Optional[int]							=  nn.functional.log_softmax(__lowercase ,       dim=-1  )
									else:
												# construct weights and biases
												a__       :						Union[str, Any]							=  [], []
												for i in range(len(self.cutoffs  )  ):
															if self.div_val == 1:
																		a__       :						Tuple							=  self.cutoff_ends[i], self.cutoff_ends[i + 1]
																		a__       :						Any							=  self.out_layers[0].weight[l_idx:r_idx]
																		a__       :						Dict							=  self.out_layers[0].bias[l_idx:r_idx]
															else:
																		a__       :						int							=  self.out_layers[i].weight
																		a__       :						Tuple							=  self.out_layers[i].bias

															if i == 0:
																		a__       :						List[str]							=  torch.cat([weight_i, self.cluster_weight] ,       dim=0  )
																		a__       :						Any							=  torch.cat([bias_i, self.cluster_bias] ,       dim=0  )

															weights.append(__lowercase  )
															biases.append(__lowercase  )

												a__       :						Optional[int]							=  weights[0], biases[0], self.out_projs[0]

												a__       :						Optional[int]							=  self._compute_logit(__lowercase ,       __lowercase ,       __lowercase ,       __lowercase  )
												a__       :						Union[str, Any]							=  nn.functional.log_softmax(__lowercase ,       dim=1  )

												if labels is None:
															a__       :						Dict							=  hidden.new_empty((head_logit.size(0  ), self.n_token)  )
												else:
															a__       :						int							=  torch.zeros_like(__lowercase ,       dtype=hidden.dtype ,       device=hidden.device  )

												a__       :						Any							=  0
												a__       :						Optional[Any]							=  [0] + self.cutoffs
												for i in range(len(__lowercase  ) - 1  ):
															a__       :						Tuple							=  cutoff_values[i], cutoff_values[i + 1]

															if labels is not None:
																		a__       :						List[str]							=  (labels >= l_idx) & (labels < r_idx)
																		a__       :						Any							=  mask_i.nonzero().squeeze()

																		if indices_i.numel() == 0:
																					continue

																		a__       :						Dict							=  labels.index_select(0 ,       __lowercase  ) - l_idx
																		a__       :						int							=  head_logprob.index_select(0 ,       __lowercase  )
																		a__       :						Tuple							=  hidden.index_select(0 ,       __lowercase  )
															else:
																		a__       :						List[Any]							=  hidden

															if i == 0:
																		if labels is not None:
																					a__       :						Union[str, Any]							=  head_logprob_i.gather(1 ,       target_i[:, None]  ).squeeze(1  )
																		else:
																					a__       :						Optional[int]							=  head_logprob[:, : self.cutoffs[0]]
															else:
																		a__       :						str							=  weights[i], biases[i], self.out_projs[i]

																		a__       :						List[str]							=  self._compute_logit(__lowercase ,       __lowercase ,       __lowercase ,       __lowercase  )
																		a__       :						List[str]							=  nn.functional.log_softmax(__lowercase ,       dim=1  )
																		a__       :						List[str]							=  self.cutoffs[0] + i - 1  # No probability for the head cluster
																		if labels is not None:
																					a__       :						Optional[int]							=  head_logprob_i[:, cluster_prob_idx] + tail_logprob_i.gather(
																					    1 ,       target_i[:, None]  ).squeeze(1  )
																		else:
																					a__       :						Optional[Any]							=  head_logprob[:, cluster_prob_idx, None] + tail_logprob_i
																					a__       :						str							=  logprob_i

															if labels is not None:
																		if (hasattr(self ,       """keep_order"""  ) and self.keep_order) or keep_order:
																					out.index_copy_(0 ,       __lowercase ,       -logprob_i  )
																		else:
																					out[offset : offset + logprob_i.size(0  )].copy_(-logprob_i  )
																		offset += logprob_i.size(0  )

									return out

						def 							SCREAMING_SNAKE_CASE__(       self ,       __lowercase  ) ->				Union[str, Any]:
									"""simple docstring"""




									if self.n_clusters == 0:
												a__       :						Union[str, Any]							=  self._compute_logit(__lowercase ,       self.out_layers[0].weight ,       self.out_layers[0].bias ,       self.out_projs[0]  )
												return nn.functional.log_softmax(__lowercase ,       dim=-1  )
									else:
												# construct weights and biases
												a__       :						Any							=  [], []
												for i in range(len(self.cutoffs  )  ):
															if self.div_val == 1:
																		a__       :						List[str]							=  self.cutoff_ends[i], self.cutoff_ends[i + 1]
																		a__       :						Dict							=  self.out_layers[0].weight[l_idx:r_idx]
																		a__       :						Any							=  self.out_layers[0].bias[l_idx:r_idx]
															else:
																		a__       :						List[Any]							=  self.out_layers[i].weight
																		a__       :						Any							=  self.out_layers[i].bias

															if i == 0:
																		a__       :						Dict							=  torch.cat([weight_i, self.cluster_weight] ,       dim=0  )
																		a__       :						Tuple							=  torch.cat([bias_i, self.cluster_bias] ,       dim=0  )

															weights.append(__lowercase  )
															biases.append(__lowercase  )

												a__       :						List[str]							=  weights[0], biases[0], self.out_projs[0]
												a__       :						Optional[Any]							=  self._compute_logit(__lowercase ,       __lowercase ,       __lowercase ,       __lowercase  )

												a__       :						Tuple							=  hidden.new_empty((head_logit.size(0  ), self.n_token)  )
												a__       :						int							=  nn.functional.log_softmax(__lowercase ,       dim=1  )

												a__       :						List[str]							=  [0] + self.cutoffs
												for i in range(len(__lowercase  ) - 1  ):
															a__       :						Union[str, Any]							=  cutoff_values[i], cutoff_values[i + 1]

															if i == 0:
																		a__       :						List[str]							=  head_logprob[:, : self.cutoffs[0]]
															else:
																		a__       :						Tuple							=  weights[i], biases[i], self.out_projs[i]

																		a__       :						Union[str, Any]							=  self._compute_logit(__lowercase ,       __lowercase ,       __lowercase ,       __lowercase  )
																		a__       :						Tuple							=  nn.functional.log_softmax(__lowercase ,       dim=1  )

																		a__       :						List[str]							=  head_logprob[:, -i] + tail_logprob_i
																		a__       :						str							=  logprob_i

												return out








import numpy as np
import torch
from torch.utils.data import Dataset

from utils import logger

class        snake_case__    (A__		):







						"""simple docstring"""




						def __init__(       self ,       __lowercase ,       __lowercase  ) ->				int:
									"""simple docstring"""




									a__       :						Tuple							=  params

									a__       :						str							=  np.array(__lowercase  )
									a__       :						List[Any]							=  np.array([len(__lowercase  ) for t in data]  )

									self.check()
									self.remove_long_sequences()
									self.remove_empty_sequences()
									self.remove_unknown_sequences()
									self.check()
									self.print_statistics()

						def __getitem__(       self ,       __lowercase  ) ->				Any:
									"""simple docstring"""




									return (self.token_ids[index], self.lengths[index])

						def __len__(       self  ) ->				Dict:
									"""simple docstring"""




									return len(self.lengths  )

						def 							SCREAMING_SNAKE_CASE__(       self  ) ->				Optional[int]:
									"""simple docstring"""




									assert len(self.token_ids  ) == len(self.lengths  )
									assert all(self.lengths[i] == len(self.token_ids[i]  ) for i in range(len(self.lengths  )  )  )

						def 							SCREAMING_SNAKE_CASE__(       self  ) ->				Tuple:
									"""simple docstring"""




									a__       :						int							=  self.params.max_model_input_size
									a__       :						int							=  self.lengths > max_len
									logger.info(F'''Splitting {sum(__lowercase  )} too long sequences.'''  )

									def divide_chunks(__lowercase ,       __lowercase  ):
												return [l[i : i + n] for i in range(0 ,       len(__lowercase  ) ,       __lowercase  )]

									a__       :						Any							=  []
									a__       :						Optional[int]							=  []
									if self.params.mlm:
												a__	,						a__       :						Any							=  self.params.special_tok_ids["""cls_token"""], self.params.special_tok_ids["""sep_token"""]
									else:
												a__	,						a__       :						Dict							=  self.params.special_tok_ids["""bos_token"""], self.params.special_tok_ids["""eos_token"""]

									for seq_, len_ in zip(self.token_ids ,       self.lengths  ):
												assert (seq_[0] == cls_id) and (seq_[-1] == sep_id), seq_
												if len_ <= max_len:
															new_tok_ids.append(seq_  )
															new_lengths.append(len_  )
												else:
															a__       :						int							=  []
															for sub_s in divide_chunks(seq_ ,       max_len - 2  ):
																		if sub_s[0] != cls_id:
																					a__       :						str							=  np.insert(__lowercase ,       0 ,       __lowercase  )
																		if sub_s[-1] != sep_id:
																					a__       :						List[str]							=  np.insert(__lowercase ,       len(__lowercase  ) ,       __lowercase  )
																		assert len(__lowercase  ) <= max_len
																		assert (sub_s[0] == cls_id) and (sub_s[-1] == sep_id), sub_s
																		sub_seqs.append(__lowercase  )

															new_tok_ids.extend(__lowercase  )
															new_lengths.extend([len(__lowercase  ) for l in sub_seqs]  )

									a__       :						Optional[int]							=  np.array(__lowercase  )
									a__       :						Any							=  np.array(__lowercase  )

						def 							SCREAMING_SNAKE_CASE__(       self  ) ->				Optional[Any]:
									"""simple docstring"""




									a__       :						Union[str, Any]							=  len(self  )
									a__       :						List[str]							=  self.lengths > 1_1
									a__       :						Dict							=  self.token_ids[indices]
									a__       :						List[str]							=  self.lengths[indices]
									a__       :						int							=  len(self  )
									logger.info(F'''Remove {init_size - new_size} too short (<=11 tokens) sequences.'''  )

						def 							SCREAMING_SNAKE_CASE__(       self  ) ->				List[Any]:
									"""simple docstring"""




									if "unk_token" not in self.params.special_tok_ids:
												return
									else:
												a__       :						Union[str, Any]							=  self.params.special_tok_ids["""unk_token"""]
									a__       :						List[Any]							=  len(self  )
									a__       :						Optional[int]							=  np.array([np.count_nonzero(a == unk_token_id  ) for a in self.token_ids]  )
									a__       :						Optional[Any]							=  (unk_occs / self.lengths) < 0.5
									a__       :						Tuple							=  self.token_ids[indices]
									a__       :						Union[str, Any]							=  self.lengths[indices]
									a__       :						Tuple							=  len(self  )
									logger.info(F'''Remove {init_size - new_size} sequences with a high level of unknown tokens (50%).'''  )

						def 							SCREAMING_SNAKE_CASE__(       self  ) ->				str:
									"""simple docstring"""




									if not self.params.is_master:
												return
									logger.info(F'''{len(self  )} sequences'''  )
									# data_len = sum(self.lengths)
									# nb_unique_tokens = len(Counter(list(chain(*self.token_ids))))
									# logger.info(f'{data_len} tokens ({nb_unique_tokens} unique)')

									# unk_idx = self.params.special_tok_ids['unk_token']
									# nb_unknown = sum([(t==unk_idx).sum() for t in self.token_ids])
									# logger.info(f'{nb_unknown} unknown tokens (covering {100*nb_unknown/data_len:.2f}% of the data)')

						def 							SCREAMING_SNAKE_CASE__(       self ,       __lowercase  ) ->				Optional[int]:
									"""simple docstring"""




									a__       :						Optional[int]							=  [t[0] for t in batch]
									a__       :						Any							=  [t[1] for t in batch]
									assert len(__lowercase  ) == len(__lowercase  )

									# Max for paddings
									a__       :						List[Any]							=  max(__lowercase  )

									# Pad token ids
									if self.params.mlm:
												a__       :						int							=  self.params.special_tok_ids["""pad_token"""]
									else:
												a__       :						List[str]							=  self.params.special_tok_ids["""unk_token"""]
									a__       :						int							=  [list(t.astype(__lowercase  )  ) + [pad_idx] * (max_seq_len_ - len(__lowercase  )) for t in token_ids]
									assert len(tk_  ) == len(__lowercase  )
									assert all(len(__lowercase  ) == max_seq_len_ for t in tk_  )

									a__       :						List[Any]							=  torch.tensor(tk_  )  # (bs, max_seq_len_)
									a__       :						Optional[int]							=  torch.tensor(__lowercase  )  # (bs)
									return tk_t, lg_t




