




import tensorflow as tf

from ...tf_utils import shape_list



class    lowercase_		(tf.keras.layers.Layer ):



						def __init__(			self	:   Optional[Any]					,							snake_case__	:   Tuple					,							snake_case__	:   Union[str, Any]					,							snake_case__	:   int					,							snake_case__	:   Optional[Any]					,							snake_case__	:   Tuple=1					,							snake_case__	:   List[str]=False					,							**snake_case__	:   Optional[int]  ):

								"""simple docstring"""

								super().__init__(**snake_case__  )

								SCREAMING_SNAKE_CASE_					      =							vocab_size
								SCREAMING_SNAKE_CASE_					      =							d_embed
								SCREAMING_SNAKE_CASE_					      =							d_proj

								SCREAMING_SNAKE_CASE_					      =							cutoffs + [vocab_size]
								SCREAMING_SNAKE_CASE_					      =							[0] + self.cutoffs
								SCREAMING_SNAKE_CASE_					      =							div_val

								SCREAMING_SNAKE_CASE_					      =							self.cutoffs[0]
								SCREAMING_SNAKE_CASE_					      =							len(self.cutoffs  ) - 1
								SCREAMING_SNAKE_CASE_					      =							self.shortlist_size + self.n_clusters
								SCREAMING_SNAKE_CASE_					      =							keep_order

								SCREAMING_SNAKE_CASE_					      =							[]
								SCREAMING_SNAKE_CASE_					      =							[]



						def 	__a					(			self	:   List[str]					,							snake_case__	:   List[str]  ):

								"""simple docstring"""

								if self.n_clusters > 0:
										SCREAMING_SNAKE_CASE_					      =							self.add_weight(
										    shape=(self.n_clusters, self.d_embed)					,							initializer='zeros'					,							trainable=snake_case__					,							name='cluster_weight'  )
										SCREAMING_SNAKE_CASE_					      =							self.add_weight(
										    shape=(self.n_clusters,)					,							initializer='zeros'					,							trainable=snake_case__					,							name='cluster_bias'  )

								if self.div_val == 1:
										for i in range(len(self.cutoffs  )  ):
												if self.d_proj != self.d_embed:
														SCREAMING_SNAKE_CASE_					      =							self.add_weight(
														    shape=(self.d_embed, self.d_proj)					,							initializer='zeros'					,							trainable=snake_case__					,							name=f'''out_projs_._{i}'''					,							)
														self.out_projs.append(snake_case__  )
												else:
														self.out_projs.append(snake_case__  )
												SCREAMING_SNAKE_CASE_					      =							self.add_weight(
												    shape=(self.vocab_size, self.d_embed)					,							initializer='zeros'					,							trainable=snake_case__					,							name=f'''out_layers_._{i}_._weight'''					,							)
												SCREAMING_SNAKE_CASE_					      =							self.add_weight(
												    shape=(self.vocab_size,)					,							initializer='zeros'					,							trainable=snake_case__					,							name=f'''out_layers_._{i}_._bias'''					,							)
												self.out_layers.append((weight, bias)  )
								else:
										for i in range(len(self.cutoffs  )  ):
												SCREAMING_SNAKE_CASE_      ,				SCREAMING_SNAKE_CASE_					      =							self.cutoff_ends[i], self.cutoff_ends[i + 1]
												SCREAMING_SNAKE_CASE_					      =							self.d_embed // (self.div_val**i)

												SCREAMING_SNAKE_CASE_					      =							self.add_weight(
												    shape=(d_emb_i, self.d_proj)					,							initializer='zeros'					,							trainable=snake_case__					,							name=f'''out_projs_._{i}'''  )
												self.out_projs.append(snake_case__  )
												SCREAMING_SNAKE_CASE_					      =							self.add_weight(
												    shape=(r_idx - l_idx, d_emb_i)					,							initializer='zeros'					,							trainable=snake_case__					,							name=f'''out_layers_._{i}_._weight'''					,							)
												SCREAMING_SNAKE_CASE_					      =							self.add_weight(
												    shape=(r_idx - l_idx,)					,							initializer='zeros'					,							trainable=snake_case__					,							name=f'''out_layers_._{i}_._bias'''					,							)
												self.out_layers.append((weight, bias)  )
								super().build(snake_case__  )



						@staticmethod
						def 	__a					(			snake_case__	:   Union[str, Any]					,							snake_case__	:   Any					,							snake_case__	:   List[str]					,							snake_case__	:   int=None  ):

								"""simple docstring"""

								SCREAMING_SNAKE_CASE_					      =							x
								if proj is not None:
										SCREAMING_SNAKE_CASE_					      =							tf.einsum('ibd,ed->ibe'					,							snake_case__					,							snake_case__  )
								return tf.einsum('ibd,nd->ibn'					,							snake_case__					,							snake_case__  ) + b



						@staticmethod
						def 	__a					(			snake_case__	:   Optional[Any]					,							snake_case__	:   int  ):

								"""simple docstring"""

								SCREAMING_SNAKE_CASE_					      =							shape_list(snake_case__  )
								SCREAMING_SNAKE_CASE_					      =							tf.range(lp_size[0]					,							dtype=target.dtype  )
								SCREAMING_SNAKE_CASE_					      =							tf.stack([r, target]					,							1  )
								return tf.gather_nd(snake_case__					,							snake_case__  )






						def 	__a					(			self	:   List[str]					,							snake_case__	:   Tuple					,							snake_case__	:   List[Any]					,							snake_case__	:   Tuple=True					,							snake_case__	:   Any=False  ):

								"""simple docstring"""

								SCREAMING_SNAKE_CASE_					      =							0
								if self.n_clusters == 0:
										SCREAMING_SNAKE_CASE_					      =							self._logit(snake_case__					,							self.out_layers[0][0]					,							self.out_layers[0][1]					,							self.out_projs[0]  )
										if target is not None:
												SCREAMING_SNAKE_CASE_					      =							tf.nn.sparse_softmax_cross_entropy_with_logits(labels=snake_case__					,							logits=snake_case__  )
										SCREAMING_SNAKE_CASE_					      =							tf.nn.log_softmax(snake_case__					,							axis=-1  )
								else:
										SCREAMING_SNAKE_CASE_					      =							shape_list(snake_case__  )
										SCREAMING_SNAKE_CASE_					      =							[]
										SCREAMING_SNAKE_CASE_					      =							tf.zeros(hidden_sizes[:2]  )
										for i in range(len(self.cutoffs  )  ):
												SCREAMING_SNAKE_CASE_      ,				SCREAMING_SNAKE_CASE_					      =							self.cutoff_ends[i], self.cutoff_ends[i + 1]
												if target is not None:
														SCREAMING_SNAKE_CASE_					      =							(target >= l_idx) & (target < r_idx)
														SCREAMING_SNAKE_CASE_					      =							tf.where(snake_case__  )
														SCREAMING_SNAKE_CASE_					      =							tf.boolean_mask(snake_case__					,							snake_case__  ) - l_idx

												if self.div_val == 1:
														SCREAMING_SNAKE_CASE_					      =							self.out_layers[0][0][l_idx:r_idx]
														SCREAMING_SNAKE_CASE_					      =							self.out_layers[0][1][l_idx:r_idx]
												else:
														SCREAMING_SNAKE_CASE_					      =							self.out_layers[i][0]
														SCREAMING_SNAKE_CASE_					      =							self.out_layers[i][1]

												if i == 0:
														SCREAMING_SNAKE_CASE_					      =							tf.concat([cur_W, self.cluster_weight]					,							0  )
														SCREAMING_SNAKE_CASE_					      =							tf.concat([cur_b, self.cluster_bias]					,							0  )

														SCREAMING_SNAKE_CASE_					      =							self._logit(snake_case__					,							snake_case__					,							snake_case__					,							self.out_projs[0]  )
														SCREAMING_SNAKE_CASE_					      =							tf.nn.log_softmax(snake_case__  )
														out.append(head_logprob[..., : self.cutoffs[0]]  )
														if target is not None:
																SCREAMING_SNAKE_CASE_					      =							tf.boolean_mask(snake_case__					,							snake_case__  )
																SCREAMING_SNAKE_CASE_					      =							self._gather_logprob(snake_case__					,							snake_case__  )
												else:
														SCREAMING_SNAKE_CASE_					      =							self._logit(snake_case__					,							snake_case__					,							snake_case__					,							self.out_projs[i]  )
														SCREAMING_SNAKE_CASE_					      =							tf.nn.log_softmax(snake_case__  )
														SCREAMING_SNAKE_CASE_					      =							self.cutoffs[0] + i - 1  # No probability for the head cluster
														SCREAMING_SNAKE_CASE_					      =							head_logprob[..., cluster_prob_idx, None] + tail_logprob
														out.append(snake_case__  )
														if target is not None:
																SCREAMING_SNAKE_CASE_					      =							tf.boolean_mask(snake_case__					,							snake_case__  )
																SCREAMING_SNAKE_CASE_					      =							tf.boolean_mask(snake_case__					,							snake_case__  )
																SCREAMING_SNAKE_CASE_					      =							self._gather_logprob(snake_case__					,							snake_case__  )
																cur_logprob += cur_head_logprob[:, self.cutoff_ends[1] + i - 1]
												if target is not None:
														loss += tf.scatter_nd(snake_case__					,							-cur_logprob					,							shape_list(snake_case__  )  )
										SCREAMING_SNAKE_CASE_					      =							tf.concat(snake_case__					,							axis=-1  )

								if target is not None:
										if return_mean:
												SCREAMING_SNAKE_CASE_					      =							tf.reduce_mean(snake_case__  )
										# Add the training-time loss value to the layer using `self.add_loss()`.
										self.add_loss(snake_case__  )

										# Log the loss as a metric (we could log arbitrary metrics,
										# including different metrics for training and inference.
										self.add_metric(snake_case__					,							name=self.name					,							aggregation='mean' if return_mean else ''  )

								return out





from collections import OrderedDict
from typing import TYPE_CHECKING, Any, List, Mapping, Optional, Union

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import TensorType, logging


if TYPE_CHECKING:
						from ...onnx.config import PatchingSpec
						from ...tokenization_utils_base import PreTrainedTokenizerBase


SCREAMING_SNAKE_CASE:     str									= logging.get_logger(__name__)

SCREAMING_SNAKE_CASE:     Optional[int]									= {
    '''allenai/longformer-base-4096''': '''https://huggingface.co/allenai/longformer-base-4096/resolve/main/config.json''',
    '''allenai/longformer-large-4096''': '''https://huggingface.co/allenai/longformer-large-4096/resolve/main/config.json''',
    '''allenai/longformer-large-4096-finetuned-triviaqa''': (
        '''https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/config.json'''
    ),
    '''allenai/longformer-base-4096-extra.pos.embd.only''': (
        '''https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/config.json'''
    ),
    '''allenai/longformer-large-4096-extra.pos.embd.only''': (
        '''https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/config.json'''
    ),
}



class    lowercase_		(SCREAMING_SNAKE_CASE__ ):
						lowerCAmelCase__			      ="longformer"



						def __init__(			self	:   Optional[Any]					,							snake_case__	:   Union[List[int], int] = 5_12					,							snake_case__	:   int = 2					,							snake_case__	:   int = 1					,							snake_case__	:   int = 0					,							snake_case__	:   int = 2					,							snake_case__	:   int = 3_05_22					,							snake_case__	:   int = 7_68					,							snake_case__	:   int = 12					,							snake_case__	:   int = 12					,							snake_case__	:   int = 30_72					,							snake_case__	:   str = "gelu"					,							snake_case__	:   float = 0.1					,							snake_case__	:   float = 0.1					,							snake_case__	:   int = 5_12					,							snake_case__	:   int = 2					,							snake_case__	:   float = 0.02					,							snake_case__	:   float = 1e-12					,							snake_case__	:   bool = False					,							**snake_case__	:   Tuple					,							):

								"""simple docstring"""

								super().__init__(pad_token_id=snake_case__					,							**snake_case__  )

								SCREAMING_SNAKE_CASE_					      =							attention_window
								SCREAMING_SNAKE_CASE_					      =							sep_token_id
								SCREAMING_SNAKE_CASE_					      =							bos_token_id
								SCREAMING_SNAKE_CASE_					      =							eos_token_id
								SCREAMING_SNAKE_CASE_					      =							vocab_size
								SCREAMING_SNAKE_CASE_					      =							hidden_size
								SCREAMING_SNAKE_CASE_					      =							num_hidden_layers
								SCREAMING_SNAKE_CASE_					      =							num_attention_heads
								SCREAMING_SNAKE_CASE_					      =							hidden_act
								SCREAMING_SNAKE_CASE_					      =							intermediate_size
								SCREAMING_SNAKE_CASE_					      =							hidden_dropout_prob
								SCREAMING_SNAKE_CASE_					      =							attention_probs_dropout_prob
								SCREAMING_SNAKE_CASE_					      =							max_position_embeddings
								SCREAMING_SNAKE_CASE_					      =							type_vocab_size
								SCREAMING_SNAKE_CASE_					      =							initializer_range
								SCREAMING_SNAKE_CASE_					      =							layer_norm_eps
								SCREAMING_SNAKE_CASE_					      =							onnx_export



class    lowercase_		(SCREAMING_SNAKE_CASE__ ):



						def __init__(			self	:   Dict					,							snake_case__	:   "PretrainedConfig"					,							snake_case__	:   str = "default"					,							snake_case__	:   "List[PatchingSpec]" = None  ):

								"""simple docstring"""

								super().__init__(snake_case__					,							snake_case__					,							snake_case__  )
								SCREAMING_SNAKE_CASE_					      =							True



						@property
						def 	__a					(			self	:   Union[str, Any]  ):

								"""simple docstring"""

								if self.task == "multiple-choice":
										SCREAMING_SNAKE_CASE_					      =							{0: 'batch', 1: 'choice', 2: 'sequence'}
								else:
										SCREAMING_SNAKE_CASE_					      =							{0: 'batch', 1: 'sequence'}
								return OrderedDict(
								    [
								        ('input_ids', dynamic_axis),
								        ('attention_mask', dynamic_axis),
								        ('global_attention_mask', dynamic_axis),
								    ]  )



						@property
						def 	__a					(			self	:   Optional[int]  ):

								"""simple docstring"""

								SCREAMING_SNAKE_CASE_					      =							super().outputs
								if self.task == "default":
										SCREAMING_SNAKE_CASE_					      =							{0: 'batch'}
								return outputs



						@property
						def 	__a					(			self	:   Optional[int]  ):

								"""simple docstring"""

								return 1e-4



						@property
						def 	__a					(			self	:   List[str]  ):

								"""simple docstring"""

								return max(super().default_onnx_opset					,							14  )






						def 	__a					(			self	:   List[str]					,							snake_case__	:   "PreTrainedTokenizerBase"					,							snake_case__	:   int = -1					,							snake_case__	:   int = -1					,							snake_case__	:   bool = False					,							snake_case__	:   Optional[TensorType] = None					,							):

								"""simple docstring"""

								SCREAMING_SNAKE_CASE_					      =							super().generate_dummy_inputs(
								    preprocessor=snake_case__					,							batch_size=snake_case__					,							seq_length=snake_case__					,							is_pair=snake_case__					,							framework=snake_case__  )
								import torch

								# for some reason, replacing this code by inputs["global_attention_mask"] = torch.randint(2, inputs["input_ids"].shape, dtype=torch.int64)
								# makes the export fail randomly
								SCREAMING_SNAKE_CASE_					      =							torch.zeros_like(inputs['input_ids']  )
								# make every second token global
								SCREAMING_SNAKE_CASE_					      =							1

								return inputs