




"""simple docstring"""







import os
import sys
import tempfile

import torch

from .state import AcceleratorState
from .utils import PrecisionType, PrepareForLaunch, is_mps_available, patch_environment





def 							lowercase					(snake_case__   : Optional[int]      ,       snake_case__   : Optional[int]=()      ,       snake_case__   : Tuple=None      ,       snake_case__   : str="no"      ,       snake_case__   : Dict="29500" )     ->  Optional[int]:


				'''simple docstring'''






				lowerCAmelCase  =				False
				lowerCAmelCase  =				False
				if any(key.startswith("""KAGGLE""" ) for key in os.environ.keys() ):
								lowerCAmelCase  =				True
				elif "IPython" in sys.modules:
								lowerCAmelCase  =				"""google.colab""" in str(sys.modules["""IPython"""].get_ipython() )

				try:
								lowerCAmelCase  =				PrecisionType(mixed_precision.lower() )
				except ValueError:
								raise ValueError(
								    f'''Unknown mixed_precision mode: {args.mixed_precision.lower()}. Choose between {PrecisionType.list()}.''' )

				if (in_colab or in_kaggle) and (os.environ.get("""TPU_NAME"""      ,       snake_case__ ) is not None):
								# TPU launch
								import torch_xla.distributed.xla_multiprocessing as xmp

								if len(AcceleratorState._shared_state ) > 0:
												raise ValueError(
												    """To train on TPU in Colab or Kaggle Kernel, the `Accelerator` should only be initialized inside """
												    """your training function. Restart your notebook and make sure no cells initializes an """
												    """`Accelerator`.""" )
								if num_processes is None:
												lowerCAmelCase  =				8

								lowerCAmelCase  =				PrepareForLaunch(snake_case__      ,       distributed_type="""TPU""" )
								print(f'''Launching a training on {num_processes} TPU cores.''' )
								xmp.spawn(snake_case__      ,       args=snake_case__      ,       nprocs=snake_case__      ,       start_method="""fork""" )
				elif in_colab:
								# No need for a distributed launch otherwise as it's either CPU or one GPU.
								if torch.cuda.is_available():
												print("""Launching training on one GPU.""" )
								else:
												print("""Launching training on one CPU.""" )
								function(*snake_case__ )
				else:
								if num_processes is None:
												raise ValueError(
												    """You have to specify the number of GPUs you would like to use, add `num_processes=...` to your call.""" )

								if num_processes > 1:
												# Multi-GPU launch
												from torch.multiprocessing import start_processes
												from torch.multiprocessing.spawn import ProcessRaisedException

												if len(AcceleratorState._shared_state ) > 0:
																raise ValueError(
																    """To launch a multi-GPU training from your notebook, the `Accelerator` should only be initialized """
																    """inside your training function. Restart your notebook and make sure no cells initializes an """
																    """`Accelerator`.""" )

												if torch.cuda.is_initialized():
																raise ValueError(
																    """To launch a multi-GPU training from your notebook, you need to avoid running any instruction """
																    """using `torch.cuda` in any cell. Restart your notebook and make sure no cells use any CUDA """
																    """function.""" )

												# torch.distributed will expect a few environment variable to be here. We set the ones common to each
												# process here (the other ones will be set be the launcher).
												with patch_environment(
												    world_size=snake_case__      ,       master_addr="""127.0.01"""      ,       master_port=snake_case__      ,       mixed_precision=snake_case__ ):
																lowerCAmelCase  =				PrepareForLaunch(snake_case__      ,       distributed_type="""MULTI_GPU""" )
																print(f'''Launching training on {num_processes} GPUs.''' )
																try:
																				start_processes(snake_case__      ,       args=snake_case__      ,       nprocs=snake_case__      ,       start_method="""fork""" )
																except ProcessRaisedException as e:
																				if "Cannot re-initialize CUDA in forked subprocess" in e.args[0]:
																								raise RuntimeError(
																								    """CUDA has been initialized before the `notebook_launcher` could create a forked subprocess. """
																								    """This likely stems from an outside import causing issues once the `notebook_launcher()` is called. """
																								    """Please review your imports and test them when running the `notebook_launcher()` to identify """
																								    """which one is problematic.""" ) from e

								else:
												# No need for a distributed launch otherwise as it's either CPU, GPU or MPS.
												if is_mps_available():
																lowerCAmelCase  =				"""1"""
																print("""Launching training on MPS.""" )
												elif torch.cuda.is_available():
																print("""Launching training on one GPU.""" )
												else:
																print("""Launching training on CPU.""" )
												function(*snake_case__ )






def 							lowercase					(snake_case__   : Optional[int]      ,       snake_case__   : List[str]=()      ,       snake_case__   : Any=2 )     ->  Tuple:


				'''simple docstring'''






				from torch.multiprocessing import start_processes

				with tempfile.NamedTemporaryFile() as tmp_file:
								# torch.distributed will expect a few environment variable to be here. We set the ones common to each
								# process here (the other ones will be set be the launcher).
								with patch_environment(
								    world_size=snake_case__      ,       master_addr="""127.0.01"""      ,       master_port="""29500"""      ,       accelerate_mixed_precision="""no"""      ,       accelerate_debug_rdv_file=tmp_file.name      ,       accelerate_use_cpu="""yes"""      ,       ):
												lowerCAmelCase  =				PrepareForLaunch(snake_case__      ,       debug=snake_case__ )
												start_processes(snake_case__      ,       args=snake_case__      ,       nprocs=snake_case__      ,       start_method="""fork""" )











"""simple docstring"""







import flax.linen as nn
import jax.numpy as jnp

from .attention_flax import FlaxTransformeraDModel
from .resnet_flax import FlaxDownsampleaD, FlaxResnetBlockaD, FlaxUpsampleaD






class       SCREAMING_SNAKE_CASE__					(					nn.Module					):
						_a    						=						42
						_a    						=						42
						_a    						=						0.0
						_a    						=						1
						_a    						=						1
						_a    						=						True
						_a    						=						False
						_a    						=						False
						_a    						=						False
						_a    						=						jnp.floataa


						def   __lowercase (							self				:			List[str]					):
										lowerCAmelCase  =				[]
										lowerCAmelCase  =				[]

										for i in range(self.num_layers					):
														lowerCAmelCase  =				self.in_channels if i == 0 else self.out_channels

														lowerCAmelCase  =				FlaxResnetBlockaD(
														    in_channels=lowerCAmelCase						,						out_channels=self.out_channels						,						dropout_prob=self.dropout						,						dtype=self.dtype						,						)
														resnets.append(lowerCAmelCase					)

														lowerCAmelCase  =				FlaxTransformeraDModel(
														    in_channels=self.out_channels						,						n_heads=self.num_attention_heads						,						d_head=self.out_channels // self.num_attention_heads						,						depth=1						,						use_linear_projection=self.use_linear_projection						,						only_cross_attention=self.only_cross_attention						,						use_memory_efficient_attention=self.use_memory_efficient_attention						,						dtype=self.dtype						,						)
														attentions.append(lowerCAmelCase					)

										lowerCAmelCase  =				resnets
										lowerCAmelCase  =				attentions

										if self.add_downsample:
														lowerCAmelCase  =				FlaxDownsampleaD(self.out_channels						,						dtype=self.dtype					)




						def __call__(							self				:			Tuple						,						lowerCAmelCase				:			str						,						lowerCAmelCase				:			Union[str, Any]						,						lowerCAmelCase				:			Tuple						,						lowerCAmelCase				:			int=True					):
										lowerCAmelCase  =				()

										for resnet, attn in zip(self.resnets						,						self.attentions					):
														lowerCAmelCase  =				resnet(lowerCAmelCase						,						lowerCAmelCase						,						deterministic=lowerCAmelCase					)
														lowerCAmelCase  =				attn(lowerCAmelCase						,						lowerCAmelCase						,						deterministic=lowerCAmelCase					)
														output_states += (hidden_states,)

										if self.add_downsample:
														lowerCAmelCase  =				self.downsamplers_a(lowerCAmelCase					)
														output_states += (hidden_states,)

										return hidden_states, output_states






class       SCREAMING_SNAKE_CASE__					(					nn.Module					):
						_a    						=						42
						_a    						=						42
						_a    						=						0.0
						_a    						=						1
						_a    						=						True
						_a    						=						jnp.floataa


						def   __lowercase (							self				:			Union[str, Any]					):
										lowerCAmelCase  =				[]

										for i in range(self.num_layers					):
														lowerCAmelCase  =				self.in_channels if i == 0 else self.out_channels

														lowerCAmelCase  =				FlaxResnetBlockaD(
														    in_channels=lowerCAmelCase						,						out_channels=self.out_channels						,						dropout_prob=self.dropout						,						dtype=self.dtype						,						)
														resnets.append(lowerCAmelCase					)
										lowerCAmelCase  =				resnets

										if self.add_downsample:
														lowerCAmelCase  =				FlaxDownsampleaD(self.out_channels						,						dtype=self.dtype					)




						def __call__(							self				:			int						,						lowerCAmelCase				:			Any						,						lowerCAmelCase				:			Any						,						lowerCAmelCase				:			List[str]=True					):
										lowerCAmelCase  =				()

										for resnet in self.resnets:
														lowerCAmelCase  =				resnet(lowerCAmelCase						,						lowerCAmelCase						,						deterministic=lowerCAmelCase					)
														output_states += (hidden_states,)

										if self.add_downsample:
														lowerCAmelCase  =				self.downsamplers_a(lowerCAmelCase					)
														output_states += (hidden_states,)

										return hidden_states, output_states






class       SCREAMING_SNAKE_CASE__					(					nn.Module					):
						_a    						=						42
						_a    						=						42
						_a    						=						42
						_a    						=						0.0
						_a    						=						1
						_a    						=						1
						_a    						=						True
						_a    						=						False
						_a    						=						False
						_a    						=						False
						_a    						=						jnp.floataa


						def   __lowercase (							self				:			List[str]					):
										lowerCAmelCase  =				[]
										lowerCAmelCase  =				[]

										for i in range(self.num_layers					):
														lowerCAmelCase  =				self.in_channels if (i == self.num_layers - 1) else self.out_channels
														lowerCAmelCase  =				self.prev_output_channel if i == 0 else self.out_channels

														lowerCAmelCase  =				FlaxResnetBlockaD(
														    in_channels=resnet_in_channels + res_skip_channels						,						out_channels=self.out_channels						,						dropout_prob=self.dropout						,						dtype=self.dtype						,						)
														resnets.append(lowerCAmelCase					)

														lowerCAmelCase  =				FlaxTransformeraDModel(
														    in_channels=self.out_channels						,						n_heads=self.num_attention_heads						,						d_head=self.out_channels // self.num_attention_heads						,						depth=1						,						use_linear_projection=self.use_linear_projection						,						only_cross_attention=self.only_cross_attention						,						use_memory_efficient_attention=self.use_memory_efficient_attention						,						dtype=self.dtype						,						)
														attentions.append(lowerCAmelCase					)

										lowerCAmelCase  =				resnets
										lowerCAmelCase  =				attentions

										if self.add_upsample:
														lowerCAmelCase  =				FlaxUpsampleaD(self.out_channels						,						dtype=self.dtype					)




						def __call__(							self				:			Optional[Any]						,						lowerCAmelCase				:			Tuple						,						lowerCAmelCase				:			Dict						,						lowerCAmelCase				:			List[Any]						,						lowerCAmelCase				:			Any						,						lowerCAmelCase				:			Dict=True					):
										for resnet, attn in zip(self.resnets						,						self.attentions					):
														# pop res hidden states
														lowerCAmelCase  =				res_hidden_states_tuple[-1]
														lowerCAmelCase  =				res_hidden_states_tuple[:-1]
														lowerCAmelCase  =				jnp.concatenate((hidden_states, res_hidden_states)						,						axis=-1					)

														lowerCAmelCase  =				resnet(lowerCAmelCase						,						lowerCAmelCase						,						deterministic=lowerCAmelCase					)
														lowerCAmelCase  =				attn(lowerCAmelCase						,						lowerCAmelCase						,						deterministic=lowerCAmelCase					)

										if self.add_upsample:
														lowerCAmelCase  =				self.upsamplers_a(lowerCAmelCase					)

										return hidden_states






class       SCREAMING_SNAKE_CASE__					(					nn.Module					):
						_a    						=						42
						_a    						=						42
						_a    						=						42
						_a    						=						0.0
						_a    						=						1
						_a    						=						True
						_a    						=						jnp.floataa


						def   __lowercase (							self				:			Tuple					):
										lowerCAmelCase  =				[]

										for i in range(self.num_layers					):
														lowerCAmelCase  =				self.in_channels if (i == self.num_layers - 1) else self.out_channels
														lowerCAmelCase  =				self.prev_output_channel if i == 0 else self.out_channels

														lowerCAmelCase  =				FlaxResnetBlockaD(
														    in_channels=resnet_in_channels + res_skip_channels						,						out_channels=self.out_channels						,						dropout_prob=self.dropout						,						dtype=self.dtype						,						)
														resnets.append(lowerCAmelCase					)

										lowerCAmelCase  =				resnets

										if self.add_upsample:
														lowerCAmelCase  =				FlaxUpsampleaD(self.out_channels						,						dtype=self.dtype					)




						def __call__(							self				:			Optional[Any]						,						lowerCAmelCase				:			Optional[Any]						,						lowerCAmelCase				:			List[str]						,						lowerCAmelCase				:			List[str]						,						lowerCAmelCase				:			Dict=True					):
										for resnet in self.resnets:
														# pop res hidden states
														lowerCAmelCase  =				res_hidden_states_tuple[-1]
														lowerCAmelCase  =				res_hidden_states_tuple[:-1]
														lowerCAmelCase  =				jnp.concatenate((hidden_states, res_hidden_states)						,						axis=-1					)

														lowerCAmelCase  =				resnet(lowerCAmelCase						,						lowerCAmelCase						,						deterministic=lowerCAmelCase					)

										if self.add_upsample:
														lowerCAmelCase  =				self.upsamplers_a(lowerCAmelCase					)

										return hidden_states






class       SCREAMING_SNAKE_CASE__					(					nn.Module					):
						_a    						=						42
						_a    						=						0.0
						_a    						=						1
						_a    						=						1
						_a    						=						False
						_a    						=						False
						_a    						=						jnp.floataa


						def   __lowercase (							self				:			List[Any]					):
										# there is always at least one resnet
										lowerCAmelCase  =				[
										    FlaxResnetBlockaD(
										        in_channels=self.in_channels						,						out_channels=self.in_channels						,						dropout_prob=self.dropout						,						dtype=self.dtype						,						)
										]

										lowerCAmelCase  =				[]

										for _ in range(self.num_layers					):
														lowerCAmelCase  =				FlaxTransformeraDModel(
														    in_channels=self.in_channels						,						n_heads=self.num_attention_heads						,						d_head=self.in_channels // self.num_attention_heads						,						depth=1						,						use_linear_projection=self.use_linear_projection						,						use_memory_efficient_attention=self.use_memory_efficient_attention						,						dtype=self.dtype						,						)
														attentions.append(lowerCAmelCase					)

														lowerCAmelCase  =				FlaxResnetBlockaD(
														    in_channels=self.in_channels						,						out_channels=self.in_channels						,						dropout_prob=self.dropout						,						dtype=self.dtype						,						)
														resnets.append(lowerCAmelCase					)

										lowerCAmelCase  =				resnets
										lowerCAmelCase  =				attentions




						def __call__(							self				:			Optional[int]						,						lowerCAmelCase				:			List[str]						,						lowerCAmelCase				:			int						,						lowerCAmelCase				:			Dict						,						lowerCAmelCase				:			List[Any]=True					):
										lowerCAmelCase  =				self.resnets[0](lowerCAmelCase						,						lowerCAmelCase					)
										for attn, resnet in zip(self.attentions						,						self.resnets[1:]					):
														lowerCAmelCase  =				attn(lowerCAmelCase						,						lowerCAmelCase						,						deterministic=lowerCAmelCase					)
														lowerCAmelCase  =				resnet(lowerCAmelCase						,						lowerCAmelCase						,						deterministic=lowerCAmelCase					)

										return hidden_states





