
"""simple docstring"""





# Copyright 2023 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import torch

from accelerate import PartialState
from accelerate.utils.operations import broadcast, gather, gather_object, pad_across_processes, reduce



def 		_lowerCAmelCase							(							UpperCamelCase_							):
		return (torch.arange(state.num_processes							) + 1.0 + (state.num_processes * state.process_index)).to(state.device							)



def 		_lowerCAmelCase							(							UpperCamelCase_							):
		__SCREAMING_SNAKE_CASE				=					create_tensor(UpperCamelCase_							)
		__SCREAMING_SNAKE_CASE				=					gather(UpperCamelCase_							)
		assert gathered_tensor.tolist() == list(range(1    ,							state.num_processes**2 + 1							)							)



def 		_lowerCAmelCase							(							UpperCamelCase_							):
		__SCREAMING_SNAKE_CASE				=					[state.process_index]
		__SCREAMING_SNAKE_CASE				=					gather_object(UpperCamelCase_							)
		assert len(UpperCamelCase_							) == state.num_processes, f"{gathered_obj}, {len(UpperCamelCase_							)} != {state.num_processes}"
		assert gathered_obj == list(range(state.num_processes							)							), f"{gathered_obj} != {list(range(state.num_processes							)							)}"



def 		_lowerCAmelCase							(							UpperCamelCase_							):
		__SCREAMING_SNAKE_CASE				=					create_tensor(UpperCamelCase_							)
		__SCREAMING_SNAKE_CASE				=					broadcast(UpperCamelCase_							)
		assert broadcasted_tensor.shape == torch.Size([state.num_processes]							)
		assert broadcasted_tensor.tolist() == list(range(1    ,							state.num_processes + 1							)							)



def 		_lowerCAmelCase							(							UpperCamelCase_							):







		if state.is_main_process:
				__SCREAMING_SNAKE_CASE				=					torch.arange(state.num_processes + 1							).to(state.device							)
		else:
				__SCREAMING_SNAKE_CASE				=					torch.arange(state.num_processes							).to(state.device							)
		__SCREAMING_SNAKE_CASE				=					pad_across_processes(UpperCamelCase_							)
		assert padded_tensor.shape == torch.Size([state.num_processes + 1]							)
		if not state.is_main_process:
				assert padded_tensor.tolist() == list(range(0    ,							state.num_processes							)							) + [0]



def 		_lowerCAmelCase							(							UpperCamelCase_							):







		if state.num_processes != 2:
				return
		__SCREAMING_SNAKE_CASE				=					create_tensor(UpperCamelCase_							)
		__SCREAMING_SNAKE_CASE				=					reduce(UpperCamelCase_    ,							"""sum"""							)
		__SCREAMING_SNAKE_CASE				=					torch.tensor([4.0, 6]							).to(state.device							)
		assert torch.allclose(UpperCamelCase_    ,							UpperCamelCase_							), f"{reduced_tensor} != {truth_tensor}"



def 		_lowerCAmelCase							(							UpperCamelCase_							):







		if state.num_processes != 2:
				return
		__SCREAMING_SNAKE_CASE				=					create_tensor(UpperCamelCase_							)
		__SCREAMING_SNAKE_CASE				=					reduce(UpperCamelCase_    ,							"""mean"""							)
		__SCREAMING_SNAKE_CASE				=					torch.tensor([2.0, 3]							).to(state.device							)
		assert torch.allclose(UpperCamelCase_    ,							UpperCamelCase_							), f"{reduced_tensor} != {truth_tensor}"



def 		_lowerCAmelCase							(							UpperCamelCase_							):
		main()



def 		_lowerCAmelCase							(							):
		__SCREAMING_SNAKE_CASE				=					PartialState()
		state.print(f"State: {state}"							)
		state.print("""testing gather"""							)
		test_gather(UpperCamelCase_							)
		state.print("""testing gather_object"""							)
		test_gather_object(UpperCamelCase_							)
		state.print("""testing broadcast"""							)
		test_broadcast(UpperCamelCase_							)
		state.print("""testing pad_across_processes"""							)
		test_pad_across_processes(UpperCamelCase_							)
		state.print("""testing reduce_sum"""							)
		test_reduce_sum(UpperCamelCase_							)
		state.print("""testing reduce_mean"""							)
		test_reduce_mean(UpperCamelCase_							)


if __name__ == "__main__":
							main()










from __future__ import annotations

import unittest

from transformers import is_tf_available
from transformers.testing_utils import require_sentencepiece, require_tf, require_tokenizers, slow


if is_tf_available():
				import tensorflow as tf

				from transformers import AutoTokenizer, TFAutoModelForSeqaSeqLM



@require_tf
@require_sentencepiece
@require_tokenizers
class       a_       (     unittest.TestCase     ):





	@slow
	def 						A_(							self      )  ->					str:




				"""simple docstring"""



				SCREAMING_SNAKE_CASE_								=						TFAutoModelForSeqaSeqLM.from_pretrained('google/mt5-small'      )
				SCREAMING_SNAKE_CASE_								=						AutoTokenizer.from_pretrained('google/mt5-small'      )

				SCREAMING_SNAKE_CASE_								=						tokenizer('Hello there'      ,     return_tensors='tf'      ).input_ids
				SCREAMING_SNAKE_CASE_								=						tokenizer('Hi I am'      ,     return_tensors='tf'      ).input_ids

				SCREAMING_SNAKE_CASE_								=						model(SCREAMING_SNAKE_CASE      ,     labels=SCREAMING_SNAKE_CASE      ).loss
				SCREAMING_SNAKE_CASE_								=						-tf.math.reduce_mean(SCREAMING_SNAKE_CASE      ).numpy()

				SCREAMING_SNAKE_CASE_								=						-2_1.2_2_8_1_6_8
				self.assertTrue(abs(mtf_score - EXPECTED_SCORE      ) < 2e-4      )
