





import os
import sys
import tempfile

import torch

from .state import AcceleratorState
from .utils import PrecisionType, PrepareForLaunch, is_mps_available, patch_environment


def 			SCREAMING_SNAKE_CASE (			_UpperCAmelCase		,      _UpperCAmelCase=()		,      _UpperCAmelCase=None		,      _UpperCAmelCase="no"		,      _UpperCAmelCase="29500"					)     ->	Optional[int]:
    lowerCamelCase__       : Tuple   					=						False
    lowerCamelCase__       : Any   					=						False
    if any(key.startswith('KAGGLE'					) for key in os.environ.keys()					):
        lowerCamelCase__       : Dict   					=						True
    elif "IPython" in sys.modules:
        lowerCamelCase__       : Dict   					=						'google.colab' in str(sys.modules['IPython'].get_ipython()					)

    try:
        lowerCamelCase__       : List[Any]   					=						PrecisionType(mixed_precision.lower()					)
    except ValueError:
        raise ValueError(
            F"""Unknown mixed_precision mode: {args.mixed_precision.lower()}. Choose between {PrecisionType.list()}."""					)

    if (in_colab or in_kaggle) and (os.environ.get('TPU_NAME'		,      _UpperCAmelCase					) is not None):
        # TPU launch
        import torch_xla.distributed.xla_multiprocessing as xmp

        if len(AcceleratorState._shared_state					) > 0:
            raise ValueError(
                'To train on TPU in Colab or Kaggle Kernel, the `Accelerator` should only be initialized inside '
                'your training function. Restart your notebook and make sure no cells initializes an '
                '`Accelerator`.'					)
        if num_processes is None:
            lowerCamelCase__       : Dict   					=						8

        lowerCamelCase__       : Union[str, Any]   					=						PrepareForLaunch(_UpperCAmelCase		,      distributed_type='TPU'					)
        print(F"""Launching a training on {num_processes} TPU cores."""					)
        xmp.spawn(_UpperCAmelCase		,      args=_UpperCAmelCase		,      nprocs=_UpperCAmelCase		,      start_method='fork'					)
    elif in_colab:
        # No need for a distributed launch otherwise as it's either CPU or one GPU.
        if torch.cuda.is_available():
            print('Launching training on one GPU.'					)
        else:
            print('Launching training on one CPU.'					)
        function(*_UpperCAmelCase					)
    else:
        if num_processes is None:
            raise ValueError(
                'You have to specify the number of GPUs you would like to use, add `num_processes=...` to your call.'					)

        if num_processes > 1:
            # Multi-GPU launch
            from torch.multiprocessing import start_processes
            from torch.multiprocessing.spawn import ProcessRaisedException

            if len(AcceleratorState._shared_state					) > 0:
                raise ValueError(
                    'To launch a multi-GPU training from your notebook, the `Accelerator` should only be initialized '
                    'inside your training function. Restart your notebook and make sure no cells initializes an '
                    '`Accelerator`.'					)

            if torch.cuda.is_initialized():
                raise ValueError(
                    'To launch a multi-GPU training from your notebook, you need to avoid running any instruction '
                    'using `torch.cuda` in any cell. Restart your notebook and make sure no cells use any CUDA '
                    'function.'					)

            # torch.distributed will expect a few environment variable to be here. We set the ones common to each
            # process here (the other ones will be set be the launcher).
            with patch_environment(
                world_size=_UpperCAmelCase		,      master_addr='127.0.01'		,      master_port=_UpperCAmelCase		,      mixed_precision=_UpperCAmelCase					):
                lowerCamelCase__       : Tuple   					=						PrepareForLaunch(_UpperCAmelCase		,      distributed_type='MULTI_GPU'					)
                print(F"""Launching training on {num_processes} GPUs."""					)
                try:
                    start_processes(_UpperCAmelCase		,      args=_UpperCAmelCase		,      nprocs=_UpperCAmelCase		,      start_method='fork'					)
                except ProcessRaisedException as e:
                    if "Cannot re-initialize CUDA in forked subprocess" in e.args[0]:
                        raise RuntimeError(
                            'CUDA has been initialized before the `notebook_launcher` could create a forked subprocess. '
                            'This likely stems from an outside import causing issues once the `notebook_launcher()` is called. '
                            'Please review your imports and test them when running the `notebook_launcher()` to identify '
                            'which one is problematic.'					) from e

        else:
            # No need for a distributed launch otherwise as it's either CPU, GPU or MPS.
            if is_mps_available():
                lowerCamelCase__       : int   					=						'1'
                print('Launching training on MPS.'					)
            elif torch.cuda.is_available():
                print('Launching training on one GPU.'					)
            else:
                print('Launching training on CPU.'					)
            function(*_UpperCAmelCase					)


def 			SCREAMING_SNAKE_CASE (			_UpperCAmelCase		,      _UpperCAmelCase=()		,      _UpperCAmelCase=2					)     ->	Optional[Any]:
    from torch.multiprocessing import start_processes

    with tempfile.NamedTemporaryFile() as tmp_file:
        # torch.distributed will expect a few environment variable to be here. We set the ones common to each
        # process here (the other ones will be set be the launcher).
        with patch_environment(
            world_size=_UpperCAmelCase		,      master_addr='127.0.01'		,      master_port='29500'		,      accelerate_mixed_precision='no'		,      accelerate_debug_rdv_file=tmp_file.name		,      accelerate_use_cpu='yes'		,      ):
            lowerCamelCase__       : List[str]   					=						PrepareForLaunch(_UpperCAmelCase		,      debug=_UpperCAmelCase					)
            start_processes(_UpperCAmelCase		,      args=_UpperCAmelCase		,      nprocs=_UpperCAmelCase		,      start_method='fork'					)












from __future__ import annotations

import unittest

from transformers import is_tf_available
from transformers.testing_utils import require_sentencepiece, require_tf, require_tokenizers, slow


if is_tf_available():
    import tensorflow as tf

    from transformers import AutoTokenizer, TFAutoModelForSeqaSeqLM

@require_tf
@require_sentencepiece
@require_tokenizers
class   lowerCAmelCase				(   unittest.TestCase     ):
 @slow
 def 	A_      ( self    : int      )     ->      Optional[Any]:
     lowerCamelCase__       : Tuple   					=						TFAutoModelForSeqaSeqLM.from_pretrained('google/mt5-small'      )
     lowerCamelCase__       : Any   					=						AutoTokenizer.from_pretrained('google/mt5-small'      )

     lowerCamelCase__       : int   					=						tokenizer('Hello there'     ,	return_tensors='tf'      ).input_ids
     lowerCamelCase__       : int   					=						tokenizer('Hi I am'     ,	return_tensors='tf'      ).input_ids

     lowerCamelCase__       : List[str]   					=						model(UpperCAmelCase     ,	labels=UpperCAmelCase      ).loss
     lowerCamelCase__       : List[Any]   					=						-tf.math.reduce_mean(UpperCAmelCase      ).numpy()

     lowerCamelCase__       : Dict   					=						-2_1.2_2_8_1_6_8
     self.assertTrue(abs(mtf_score - EXPECTED_SCORE      ) < 2e-4      )





