





"""simple docstring"""




import warnings
from dataclasses import dataclass, field
from typing import Optional, Tuple

from .training_args import TrainingArguments
from .utils import cached_property, is_tf_available, logging, requires_backends


UpperCamelCase_           =logging.get_logger(__name__)

if is_tf_available():
						import tensorflow as tf




@dataclass
class  _a						(      _lowerCAmelCase				):
				UpperCamelCase								=    '''tf'''
				UpperCamelCase								=    field(
				    default=_lowerCAmelCase	,				metadata={'''help''': '''Name of TPU'''}	,				)

				UpperCamelCase								=    field(
				    default=_lowerCAmelCase	,				metadata={'''help''': '''Zone of TPU'''}	,				)

				UpperCamelCase								=    field(
				    default=_lowerCAmelCase	,				metadata={'''help''': '''Name of Cloud TPU-enabled project'''}	,				)

				UpperCamelCase								=    field(
				    default=1.0	,				metadata={'''help''': '''Power for the Polynomial decay LR scheduler.'''}	,				)

				UpperCamelCase								=    field(default=_lowerCAmelCase	,				metadata={'''help''': '''Whether to activate the XLA compilation or not'''}				)






				@cached_property
				def 				snake_case	( self		:    Union[str, Any]							)	->      Tuple["tf.distribute.Strategy", int]:




										'''simple docstring'''



										requires_backends(self, ['''tf''']							)
										logger.info('''Tensorflow: setting up strategy'''							)

										_UpperCamelCase			:					List[Any]     =    tf.config.list_physical_devices('''GPU'''							)

										# Set to float16 at first
										if self.fpaa:
																tf.keras.mixed_precision.set_global_policy('''mixed_float16'''							)

										if self.no_cuda:
																_UpperCamelCase			:					Tuple     =    tf.distribute.OneDeviceStrategy(device='''/cpu:0'''							)
										else:
																try:
																						if self.tpu_name:
																												_UpperCamelCase			:					Dict     =    tf.distribute.cluster_resolver.TPUClusterResolver(
																												    self.tpu_name, zone=self.tpu_zone, project=self.gcp_project							)
																						else:
																												_UpperCamelCase			:					List[str]     =    tf.distribute.cluster_resolver.TPUClusterResolver()
																except ValueError:
																						if self.tpu_name:
																												raise RuntimeError(f"""Couldn't connect to TPU {self.tpu_name}!"""							)
																						else:
																												_UpperCamelCase			:					str     =    None

																if tpu:
																						# Set to bfloat16 in case of TPU
																						if self.fpaa:
																												tf.keras.mixed_precision.set_global_policy('''mixed_bfloat16'''							)

																						tf.config.experimental_connect_to_cluster(lowerCAmelCase__							)
																						tf.tpu.experimental.initialize_tpu_system(lowerCAmelCase__							)

																						_UpperCamelCase			:					Optional[Any]     =    tf.distribute.TPUStrategy(lowerCAmelCase__							)

																elif len(lowerCAmelCase__							) == 0:
																						_UpperCamelCase			:					List[Any]     =    tf.distribute.OneDeviceStrategy(device='''/cpu:0'''							)
																elif len(lowerCAmelCase__							) == 1:
																						_UpperCamelCase			:					Tuple     =    tf.distribute.OneDeviceStrategy(device='''/gpu:0'''							)
																elif len(lowerCAmelCase__							) > 1:
																						# If you only want to use a specific subset of GPUs use `CUDA_VISIBLE_DEVICES=0`
																						_UpperCamelCase			:					Optional[Any]     =    tf.distribute.MirroredStrategy()
																else:
																						raise ValueError('''Cannot find the proper strategy, please check your environment properties.'''							)

										return strategy






				@property
				def 				snake_case	( self		:    Dict							)	->      "tf.distribute.Strategy":




										'''simple docstring'''



										requires_backends(self, ['''tf''']							)
										return self._setup_strategy






				@property
				def 				snake_case	( self		:    Optional[Any]							)	->      int:




										'''simple docstring'''



										requires_backends(self, ['''tf''']							)
										return self._setup_strategy.num_replicas_in_sync






				@property
				def 				snake_case	( self		:    Optional[int]							)	->      Tuple:




										'''simple docstring'''



										return False  # TF Logging is handled by Keras not the Trainer






				@property
				def 				snake_case	( self		:    List[str]							)	->      int:




										'''simple docstring'''



										if self.per_gpu_train_batch_size:
																logger.warning(
																    '''Using deprecated `--per_gpu_train_batch_size` argument which will be removed in a future '''
																    '''version. Using `--per_device_train_batch_size` is preferred.'''							)
										_UpperCamelCase			:					Union[str, Any]     =    self.per_gpu_train_batch_size or self.per_device_train_batch_size
										return per_device_batch_size * self.n_replicas






				@property
				def 				snake_case	( self		:    List[str]							)	->      int:




										'''simple docstring'''



										if self.per_gpu_eval_batch_size:
																logger.warning(
																    '''Using deprecated `--per_gpu_eval_batch_size` argument which will be removed in a future '''
																    '''version. Using `--per_device_eval_batch_size` is preferred.'''							)
										_UpperCamelCase			:					Dict     =    self.per_gpu_eval_batch_size or self.per_device_eval_batch_size
										return per_device_batch_size * self.n_replicas






				@property
				def 				snake_case	( self		:    str							)	->      int:




										'''simple docstring'''



										requires_backends(self, ['''tf''']							)
										warnings.warn(
										    '''The n_gpu argument is deprecated and will be removed in a future version, use n_replicas instead.''', lowerCAmelCase__, )
										return self._setup_strategy.num_replicas_in_sync






from ...configuration_utils import PretrainedConfig
from ...utils import logging


__UpperCAmelCase      = logging.get_logger(__name__)

__UpperCAmelCase      = {
    """uw-madison/nystromformer-512""": """https://huggingface.co/uw-madison/nystromformer-512/resolve/main/config.json""",
    # See all Nystromformer models at https://huggingface.co/models?filter=nystromformer
}



class SCREAMING_SNAKE_CASE	(  a_ ):
  """simple docstring"""






  lowerCamelCase    :			Optional[Any]						      ="nystromformer"







  def __init__(			self :    List[str]       ,							lowerCAmelCase :    List[Any]=3_00_00       ,							lowerCAmelCase :    Union[str, Any]=7_68       ,							lowerCAmelCase :    int=12       ,							lowerCAmelCase :    str=12       ,							lowerCAmelCase :    Tuple=30_72       ,							lowerCAmelCase :    Optional[Any]="gelu_new"       ,							lowerCAmelCase :    Any=0.1       ,							lowerCAmelCase :    Tuple=0.1       ,							lowerCAmelCase :    Tuple=5_10       ,							lowerCAmelCase :    int=2       ,							lowerCAmelCase :    Any=64       ,							lowerCAmelCase :    Dict=64       ,							lowerCAmelCase :    List[str]=65       ,							lowerCAmelCase :    Union[str, Any]=False       ,							lowerCAmelCase :    int=0.02       ,							lowerCAmelCase :    int=1e-5       ,							lowerCAmelCase :    Dict=1       ,							lowerCAmelCase :    str=0       ,							lowerCAmelCase :    List[Any]=2       ,							**lowerCAmelCase :    int       ,							)							->	List[Any]:






     """simple docstring"""







     __lowerCAmelCase  :			Optional[Any]					    =						vocab_size
     __lowerCAmelCase  :			Any					    =						max_position_embeddings
     __lowerCAmelCase  :			List[Any]					    =						hidden_size
     __lowerCAmelCase  :			Dict					    =						num_hidden_layers
     __lowerCAmelCase  :			Union[str, Any]					    =						num_attention_heads
     __lowerCAmelCase  :			int					    =						intermediate_size
     __lowerCAmelCase  :			str					    =						hidden_act
     __lowerCAmelCase  :			Any					    =						hidden_dropout_prob
     __lowerCAmelCase  :			str					    =						attention_probs_dropout_prob
     __lowerCAmelCase  :			List[str]					    =						initializer_range
     __lowerCAmelCase  :			List[str]					    =						type_vocab_size
     __lowerCAmelCase  :			Union[str, Any]					    =						segment_means_seq_len
     __lowerCAmelCase  :			Tuple					    =						num_landmarks
     __lowerCAmelCase  :			Dict					    =						conv_kernel_size
     __lowerCAmelCase  :			Tuple					    =						inv_coeff_init_option
     __lowerCAmelCase  :			List[Any]					    =						layer_norm_eps
     super().__init__(pad_token_id=lowerCAmelCase       ,							bos_token_id=lowerCAmelCase       ,							eos_token_id=lowerCAmelCase       ,							**lowerCAmelCase    )


