


'''simple docstring'''



# Copyright 2023 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import torch

from accelerate import PartialState
from accelerate.utils.operations import broadcast, gather, gather_object, pad_across_processes, reduce






def   lowerCamelCase							(_SCREAMING_SNAKE_CASE				:       int       ):
					return (torch.arange(state.num_processes       ) + 1.0 + (state.num_processes * state.process_index)).to(state.device       )






def   lowerCamelCase							(_SCREAMING_SNAKE_CASE				:       Optional[int]       ):
					__a      :	List[str]    =  create_tensor(_SCREAMING_SNAKE_CASE       )
					__a      :	List[str]    =  gather(_SCREAMING_SNAKE_CASE       )
					assert gathered_tensor.tolist() == list(range(1						,				state.num_processes**2 + 1       )       )






def   lowerCamelCase							(_SCREAMING_SNAKE_CASE				:       List[str]       ):
					__a      :	str    =  [state.process_index]
					__a      :	int    =  gather_object(_SCREAMING_SNAKE_CASE       )
					assert len(_SCREAMING_SNAKE_CASE       ) == state.num_processes, F"""{gathered_obj}, {len(_SCREAMING_SNAKE_CASE       )} != {state.num_processes}"""
					assert gathered_obj == list(range(state.num_processes       )       ), F"""{gathered_obj} != {list(range(state.num_processes       )       )}"""






def   lowerCamelCase							(_SCREAMING_SNAKE_CASE				:       int       ):
					__a      :	int    =  create_tensor(_SCREAMING_SNAKE_CASE       )
					__a      :	str    =  broadcast(_SCREAMING_SNAKE_CASE       )
					assert broadcasted_tensor.shape == torch.Size([state.num_processes]       )
					assert broadcasted_tensor.tolist() == list(range(1						,				state.num_processes + 1       )       )






def   lowerCamelCase							(_SCREAMING_SNAKE_CASE				:       str       ):
					# We need to pad the tensor with one more element if we are the main process
					# to ensure that we can pad
					if state.is_main_process:
										__a      :	str    =  torch.arange(state.num_processes + 1       ).to(state.device       )
					else:
										__a      :	Optional[Any]    =  torch.arange(state.num_processes       ).to(state.device       )
					__a      :	int    =  pad_across_processes(_SCREAMING_SNAKE_CASE       )
					assert padded_tensor.shape == torch.Size([state.num_processes + 1]       )
					if not state.is_main_process:
										assert padded_tensor.tolist() == list(range(0						,				state.num_processes       )       ) + [0]






def   lowerCamelCase							(_SCREAMING_SNAKE_CASE				:       Optional[int]       ):
					# For now runs on only two processes
					if state.num_processes != 2:
										return
					__a      :	str    =  create_tensor(_SCREAMING_SNAKE_CASE       )
					__a      :	Optional[Any]    =  reduce(_SCREAMING_SNAKE_CASE						,				'sum'       )
					__a      :	List[str]    =  torch.tensor([4.0, 6]       ).to(state.device       )
					assert torch.allclose(_SCREAMING_SNAKE_CASE						,				_SCREAMING_SNAKE_CASE       ), F"""{reduced_tensor} != {truth_tensor}"""






def   lowerCamelCase							(_SCREAMING_SNAKE_CASE				:       Dict       ):
					# For now runs on only two processes
					if state.num_processes != 2:
										return
					__a      :	int    =  create_tensor(_SCREAMING_SNAKE_CASE       )
					__a      :	List[Any]    =  reduce(_SCREAMING_SNAKE_CASE						,				'mean'       )
					__a      :	List[Any]    =  torch.tensor([2.0, 3]       ).to(state.device       )
					assert torch.allclose(_SCREAMING_SNAKE_CASE						,				_SCREAMING_SNAKE_CASE       ), F"""{reduced_tensor} != {truth_tensor}"""






def   lowerCamelCase							(_SCREAMING_SNAKE_CASE				:       List[Any]       ):
					# For xla_spawn (TPUs)
					main()






def   lowerCamelCase							():
					__a      :	Dict    =  PartialState()
					state.print(F"""State: {state}"""       )
					state.print('testing gather'       )
					test_gather(_SCREAMING_SNAKE_CASE       )
					state.print('testing gather_object'       )
					test_gather_object(_SCREAMING_SNAKE_CASE       )
					state.print('testing broadcast'       )
					test_broadcast(_SCREAMING_SNAKE_CASE       )
					state.print('testing pad_across_processes'       )
					test_pad_across_processes(_SCREAMING_SNAKE_CASE       )
					state.print('testing reduce_sum'       )
					test_reduce_sum(_SCREAMING_SNAKE_CASE       )
					state.print('testing reduce_mean'       )
					test_reduce_mean(_SCREAMING_SNAKE_CASE       )


if __name__ == "__main__":
	main()







'''simple docstring'''



from __future__ import annotations

from typing import Dict

from ...configuration_utils import PretrainedConfig


__lowercase     :						List[Any]	=     {
    'susnato/ernie-m-base_pytorch': 'https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/config.json',
    'susnato/ernie-m-large_pytorch': 'https://huggingface.co/susnato/ernie-m-large_pytorch/blob/main/config.json',
}





class 							__UpperCamelCase    (		lowerCAmelCase_ ):
	A_           =   "ernie_m"
	A_           =   {"dropout": "classifier_dropout", "num_classes": "num_labels"}




	def __init__(       self						,    __a = 25_0002						,    __a = 768						,    __a = 12						,    __a = 12						,    __a = 3072						,    __a = "gelu"						,    __a = 0.1						,    __a = 0.1						,    __a = 514						,    __a = 0.02						,    __a = 1						,    __a = 1E-0_5						,    __a=None						,    __a=False						,    __a=0.0						,    **__a						,    ):

						'''simple docstring'''





						super().__init__(pad_token_id=__a						,    **__a							)
						__a      :	int    =  vocab_size
						__a      :	Dict    =  hidden_size
						__a      :	str    =  num_hidden_layers
						__a      :	Dict    =  num_attention_heads
						__a      :	List[str]    =  intermediate_size
						__a      :	Union[str, Any]    =  hidden_act
						__a      :	List[Any]    =  hidden_dropout_prob
						__a      :	str    =  attention_probs_dropout_prob
						__a      :	Any    =  max_position_embeddings
						__a      :	int    =  initializer_range
						__a      :	Dict    =  layer_norm_eps
						__a      :	int    =  classifier_dropout
						__a      :	Dict    =  is_decoder
						__a      :	int    =  act_dropout



