





'''simple docstring'''


import os
import sys
import tempfile

import torch

from .state import AcceleratorState
from .utils import PrecisionType, PrepareForLaunch, is_mps_available, patch_environment


def 						__a(SCREAMING_SNAKE_CASE_	:       List[Any]		,							SCREAMING_SNAKE_CASE_	:       Optional[int]=()		,							SCREAMING_SNAKE_CASE_	:       Optional[int]=None		,							SCREAMING_SNAKE_CASE_	:       Optional[Any]="no"		,							SCREAMING_SNAKE_CASE_	:       Any="29500"     ):



    '''simple docstring'''





    _lowerCAmelCase						      =   False
    _lowerCAmelCase						      =   False
    if any(key.startswith("KAGGLE"     ) for key in os.environ.keys()     ):
        _lowerCAmelCase						      =   True
    elif "IPython" in sys.modules:
        _lowerCAmelCase						      =   "google.colab" in str(sys.modules["IPython"].get_ipython()     )

    try:
        _lowerCAmelCase						      =   PrecisionType(mixed_precision.lower()     )
    except ValueError:
        raise ValueError(
            F'''Unknown mixed_precision mode: {args.mixed_precision.lower()}. Choose between {PrecisionType.list()}.'''     )

    if (in_colab or in_kaggle) and (os.environ.get("TPU_NAME"		,							SCREAMING_SNAKE_CASE_     ) is not None):
        # TPU launch
        import torch_xla.distributed.xla_multiprocessing as xmp

        if len(AcceleratorState._shared_state     ) > 0:
            raise ValueError(
                "To train on TPU in Colab or Kaggle Kernel, the `Accelerator` should only be initialized inside "
                "your training function. Restart your notebook and make sure no cells initializes an "
                "`Accelerator`."     )
        if num_processes is None:
            _lowerCAmelCase						      =   8

        _lowerCAmelCase						      =   PrepareForLaunch(SCREAMING_SNAKE_CASE_		,							distributed_type="TPU"     )
        print(F'''Launching a training on {num_processes} TPU cores.'''     )
        xmp.spawn(SCREAMING_SNAKE_CASE_		,							args=SCREAMING_SNAKE_CASE_		,							nprocs=SCREAMING_SNAKE_CASE_		,							start_method="fork"     )
    elif in_colab:
        # No need for a distributed launch otherwise as it's either CPU or one GPU.
        if torch.cuda.is_available():
            print("Launching training on one GPU."     )
        else:
            print("Launching training on one CPU."     )
        function(*SCREAMING_SNAKE_CASE_     )
    else:
        if num_processes is None:
            raise ValueError(
                "You have to specify the number of GPUs you would like to use, add `num_processes=...` to your call."     )

        if num_processes > 1:
            # Multi-GPU launch
            from torch.multiprocessing import start_processes
            from torch.multiprocessing.spawn import ProcessRaisedException

            if len(AcceleratorState._shared_state     ) > 0:
                raise ValueError(
                    "To launch a multi-GPU training from your notebook, the `Accelerator` should only be initialized "
                    "inside your training function. Restart your notebook and make sure no cells initializes an "
                    "`Accelerator`."     )

            if torch.cuda.is_initialized():
                raise ValueError(
                    "To launch a multi-GPU training from your notebook, you need to avoid running any instruction "
                    "using `torch.cuda` in any cell. Restart your notebook and make sure no cells use any CUDA "
                    "function."     )

            # torch.distributed will expect a few environment variable to be here. We set the ones common to each
            # process here (the other ones will be set be the launcher).
            with patch_environment(
                world_size=SCREAMING_SNAKE_CASE_		,							master_addr="127.0.01"		,							master_port=SCREAMING_SNAKE_CASE_		,							mixed_precision=SCREAMING_SNAKE_CASE_     ):
                _lowerCAmelCase						      =   PrepareForLaunch(SCREAMING_SNAKE_CASE_		,							distributed_type="MULTI_GPU"     )
                print(F'''Launching training on {num_processes} GPUs.'''     )
                try:
                    start_processes(SCREAMING_SNAKE_CASE_		,							args=SCREAMING_SNAKE_CASE_		,							nprocs=SCREAMING_SNAKE_CASE_		,							start_method="fork"     )
                except ProcessRaisedException as e:
                    if "Cannot re-initialize CUDA in forked subprocess" in e.args[0]:
                        raise RuntimeError(
                            "CUDA has been initialized before the `notebook_launcher` could create a forked subprocess. "
                            "This likely stems from an outside import causing issues once the `notebook_launcher()` is called. "
                            "Please review your imports and test them when running the `notebook_launcher()` to identify "
                            "which one is problematic."     ) from e

        else:
            # No need for a distributed launch otherwise as it's either CPU, GPU or MPS.
            if is_mps_available():
                _lowerCAmelCase						      =   "1"
                print("Launching training on MPS."     )
            elif torch.cuda.is_available():
                print("Launching training on one GPU."     )
            else:
                print("Launching training on CPU."     )
            function(*SCREAMING_SNAKE_CASE_     )





def 						__a(SCREAMING_SNAKE_CASE_	:       List[str]		,							SCREAMING_SNAKE_CASE_	:       Optional[int]=()		,							SCREAMING_SNAKE_CASE_	:       List[str]=2     ):



    '''simple docstring'''





    from torch.multiprocessing import start_processes

    with tempfile.NamedTemporaryFile() as tmp_file:
        # torch.distributed will expect a few environment variable to be here. We set the ones common to each
        # process here (the other ones will be set be the launcher).
        with patch_environment(
            world_size=SCREAMING_SNAKE_CASE_		,							master_addr="127.0.01"		,							master_port="29500"		,							accelerate_mixed_precision="no"		,							accelerate_debug_rdv_file=tmp_file.name		,							accelerate_use_cpu="yes"		,							):
            _lowerCAmelCase						      =   PrepareForLaunch(SCREAMING_SNAKE_CASE_		,							debug=SCREAMING_SNAKE_CASE_     )
            start_processes(SCREAMING_SNAKE_CASE_		,							args=SCREAMING_SNAKE_CASE_		,							nprocs=SCREAMING_SNAKE_CASE_		,							start_method="fork"     )





'''simple docstring'''





from ...configuration_utils import PretrainedConfig
from ...utils import logging


UpperCamelCase_    						=       logging.get_logger(__name__)

UpperCamelCase_    						=       {
    """facebook/s2t-wav2vec2-large-en-de""": (
        """https://huggingface.co/facebook/s2t-wav2vec2-large-en-de/resolve/main/config.json"""
    ),
    # See all Speech2Text models at https://huggingface.co/models?filter=speech2text2
}





class 		__SCREAMING_SNAKE_CASE							(			lowercase__      ):
				lowerCamelCase_		       =    'speech_to_text_2'
				lowerCamelCase_		       =    ['past_key_values']
				lowerCamelCase_		       =    {'num_attention_heads': 'decoder_attention_heads', 'hidden_size': 'd_model'}







				def __init__(       self     :		int		,					UpperCAmelCase__     :		Dict=10000		,					UpperCAmelCase__     :		Optional[int]=6		,					UpperCAmelCase__     :		str=2048		,					UpperCAmelCase__     :		Optional[Any]=4		,					UpperCAmelCase__     :		Union[str, Any]=0.0		,					UpperCAmelCase__     :		Optional[Any]=True		,					UpperCAmelCase__     :		Optional[Any]="relu"		,					UpperCAmelCase__     :		List[str]=256		,					UpperCAmelCase__     :		Union[str, Any]=0.1		,					UpperCAmelCase__     :		List[Any]=0.0		,					UpperCAmelCase__     :		Optional[Any]=0.0		,					UpperCAmelCase__     :		Tuple=0.02		,					UpperCAmelCase__     :		int=2		,					UpperCAmelCase__     :		str=True		,					UpperCAmelCase__     :		str=1		,					UpperCAmelCase__     :		Optional[Any]=0		,					UpperCAmelCase__     :		Optional[int]=2		,					UpperCAmelCase__     :		List[Any]=1024		,					**UpperCAmelCase__     :		Dict		,					):
										'''simple docstring'''
										lowercase :							List[str]    		=vocab_size
										lowercase :							Optional[int]    		=d_model
										lowercase :							Optional[Any]    		=decoder_ffn_dim
										lowercase :							Any    		=decoder_layers
										lowercase :							Dict    		=decoder_attention_heads
										lowercase :							List[Any]    		=dropout
										lowercase :							List[Any]    		=attention_dropout
										lowercase :							Any    		=activation_dropout
										lowercase :							Optional[Any]    		=activation_function
										lowercase :							Optional[int]    		=init_std
										lowercase :							Dict    		=decoder_layerdrop
										lowercase :							Optional[int]    		=use_cache
										lowercase :							Optional[Any]    		=decoder_layers
										lowercase :							List[str]    		=scale_embedding  # scale factor will be sqrt(d_model) if True
										lowercase :							str    		=max_target_positions

										super().__init__(
										    pad_token_id=UpperCAmelCase__		,					bos_token_id=UpperCAmelCase__		,					eos_token_id=UpperCAmelCase__		,					decoder_start_token_id=UpperCAmelCase__		,					**UpperCAmelCase__		,					)





