





"""simple docstring"""





from ...configuration_utils import PretrainedConfig
from ...utils import logging


lowercase__				:Optional[int]				   =       logging.get_logger(__name__)

lowercase__				:List[str]				   =       {
    'google/switch-base-8': 'https://huggingface.co/google/switch-base-8/blob/main/config.json',
}




class 							snake_case							(			__UpperCAmelCase ):

      '''simple docstring'''



      _A     :       str         =       'switch_transformers'
      _A     :       str         =       ['past_key_values']
      _A     :       List[str]         =       {'hidden_size': 'd_model', 'num_attention_heads': 'num_heads', 'num_hidden_layers': 'num_layers'}

      def __init__(							self       :      List[str]						,				__lowercase       :      Tuple=32_128						,				__lowercase       :      Union[str, Any]=768						,				__lowercase       :      int=64						,				__lowercase       :      str=2_048						,				__lowercase       :      List[str]=64						,				__lowercase       :      List[Any]=12						,				__lowercase       :      str=3						,				__lowercase       :      List[Any]=12						,				__lowercase       :      Optional[int]=3						,				__lowercase       :      Dict=12						,				__lowercase       :      List[Any]=8						,				__lowercase       :      Optional[int]=False						,				__lowercase       :      Tuple=0.0_1						,				__lowercase       :      List[Any]="float32"						,				__lowercase       :      Tuple=False						,				__lowercase       :      Any=32						,				__lowercase       :      int=128						,				__lowercase       :      Tuple=0.1						,				__lowercase       :      Any=1e-6						,				__lowercase       :      Union[str, Any]=0.0_0_1						,				__lowercase       :      str=0.0_0_1						,				__lowercase       :      Optional[Any]=1.0						,				__lowercase       :      List[Any]="relu"						,				__lowercase       :      Tuple=True						,				__lowercase       :      Optional[Any]=False						,				__lowercase       :      int=True						,				__lowercase       :      Optional[Any]=0						,				__lowercase       :      int=1						,				**__lowercase       :      Tuple						,				):


        '''simple docstring'''




        __UpperCAmelCase    :						Any       =						vocab_size
        __UpperCAmelCase    :						Optional[int]       =						d_model
        __UpperCAmelCase    :						Optional[Any]       =						d_kv
        __UpperCAmelCase    :						str       =						d_ff

        __UpperCAmelCase    :						List[str]       =						num_sparse_encoder_layers

        __UpperCAmelCase    :						str       =						num_layers
        __UpperCAmelCase    :						Dict       =						(
            num_decoder_layers if num_decoder_layers is not None else self.num_layers
        )  # default = symmetry
        __UpperCAmelCase    :						Optional[int]       =						num_sparse_decoder_layers

        # This tells us, each how many encoder layer we'll have to set a sparse layer.
        if self.num_sparse_encoder_layers > 0:
          __UpperCAmelCase    :						List[str]       =						self.num_layers // self.num_sparse_encoder_layers
        else:
          __UpperCAmelCase    :						List[str]       =						self.num_layers  # HACK: this will create 0 sparse layers

        # This tells us, each how many encoder layer we'll have to set a sparse layer.
        if self.num_sparse_decoder_layers > 0:
          __UpperCAmelCase    :						int       =						self.num_decoder_layers // self.num_sparse_decoder_layers
        else:
          __UpperCAmelCase    :						Union[str, Any]       =						self.num_decoder_layers  # HACK: this will create 0 sparse layers

        __UpperCAmelCase    :						List[Any]       =						num_heads
        __UpperCAmelCase    :						int       =						num_experts
        __UpperCAmelCase    :						Any       =						expert_capacity
        __UpperCAmelCase    :						List[Any]       =						router_bias
        __UpperCAmelCase    :						Any       =						router_jitter_noise
        if router_dtype not in ["float32", "float16", "bfloat16"]:
          raise ValueError(f'''`router_dtype` must be one of \'float32\', \'float16\' or \'bfloat16\', got {router_dtype}''' )
        __UpperCAmelCase    :						int       =						router_dtype

        __UpperCAmelCase    :						str       =						router_ignore_padding_tokens
        __UpperCAmelCase    :						int       =						relative_attention_num_buckets
        __UpperCAmelCase    :						List[Any]       =						relative_attention_max_distance

        __UpperCAmelCase    :						Tuple       =						dropout_rate
        __UpperCAmelCase    :						Optional[int]       =						layer_norm_epsilon
        __UpperCAmelCase    :						List[str]       =						initializer_factor
        __UpperCAmelCase    :						Union[str, Any]       =						feed_forward_proj
        __UpperCAmelCase    :						str       =						use_cache
        __UpperCAmelCase    :						List[str]       =						add_router_probs

        __UpperCAmelCase    :						Dict       =						router_z_loss_coef
        __UpperCAmelCase    :						Optional[Any]       =						router_aux_loss_coef

        __UpperCAmelCase    :						Optional[Any]       =						self.feed_forward_proj.split('''-''' )
        __UpperCAmelCase    :						Union[str, Any]       =						act_info[-1]
        __UpperCAmelCase    :						int       =						act_info[0] == '''gated'''

        if len(__lowercase ) > 1 and act_info[0] != "gated" or len(__lowercase ) > 2:
          raise ValueError(
              f'''`feed_forward_proj`: {feed_forward_proj} is not a valid activation function of the dense layer.'''
              '''Please make sure `feed_forward_proj` is of the format `gated-{ACT_FN}` or `{ACT_FN}`, e.g. '''
              '''\'gated-gelu\' or \'relu\'''' )

        # for backwards compatibility
        if feed_forward_proj == "gated-gelu":
          __UpperCAmelCase    :						Tuple       =						'''gelu_new'''

        super().__init__(
            pad_token_id=__lowercase						,				eos_token_id=__lowercase						,				is_encoder_decoder=__lowercase						,				**__lowercase						,				)






"""simple docstring"""





# coding=utf-8
# Copyright 2020 The HuggingFace Inc. team.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# this script dumps information about the environment

import os
import sys

import transformers


lowercase__				:int				   =       '3'

print('Python version:', sys.version)
print('transformers version:', transformers.__version__)

try:
  import torch

  print('Torch version:', torch.__version__)
  print('Cuda available:', torch.cuda.is_available())
  print('Cuda version:', torch.version.cuda)
  print('CuDNN version:', torch.backends.cudnn.version())
  print('Number of GPUs available:', torch.cuda.device_count())
  print('NCCL version:', torch.cuda.nccl.version())
except ImportError:
  print('Torch version:', None)

try:
  import deepspeed

  print('DeepSpeed version:', deepspeed.__version__)
except ImportError:
  print('DeepSpeed version:', None)

try:
  import tensorflow as tf

  print('TensorFlow version:', tf.__version__)
  print('TF GPUs available:', bool(tf.config.list_physical_devices('GPU')))
  print('Number of TF GPUs available:', len(tf.config.list_physical_devices('GPU')))
except ImportError:
  print('TensorFlow version:', None)