






import argparse
import json
from pathlib import Path

import requests
import torch
from huggingface_hub import cached_download, hf_hub_download, hf_hub_url
from PIL import Image

from transformers import DetaConfig, DetaForObjectDetection, DetaImageProcessor, SwinConfig
from transformers.utils import logging


logging.set_verbosity_info()
A_		:		List[str]		     =			logging.get_logger(__name__)

def   snake_case (UpperCAmelCase__  )	->    str:
  UpperCamelCase_: int   				=    SwinConfig(
      embed_dim=1_9_2     ,   depths=(2, 2, 1_8, 2)     ,   num_heads=(6, 1_2, 2_4, 4_8)     ,   window_size=1_2     ,   out_features=['stage2', 'stage3', 'stage4']     ,   )

  UpperCamelCase_: List[str]   				=    DetaConfig(
      backbone_config=UpperCAmelCase__     ,   num_queries=9_0_0     ,   encoder_ffn_dim=2_0_4_8     ,   decoder_ffn_dim=2_0_4_8     ,   num_feature_levels=5     ,   assign_first_stage=UpperCAmelCase__     ,   with_box_refine=UpperCAmelCase__     ,   two_stage=UpperCAmelCase__     ,   )

  # set labels
  UpperCamelCase_: List[Any]   				=    'huggingface/label-files'
  if "o365" in model_name:
    UpperCamelCase_: str   				=    3_6_6
    UpperCamelCase_: int   				=    'object365-id2label.json'
  else:
    UpperCamelCase_: Optional[int]   				=    9_1
    UpperCamelCase_: Optional[Any]   				=    'coco-detection-id2label.json'

  UpperCamelCase_: Dict   				=    num_labels
  UpperCamelCase_: Optional[int]   				=    json.load(open(cached_download(hf_hub_url(UpperCAmelCase__     ,   UpperCAmelCase__     ,   repo_type='dataset'  )  )     ,   'r'  )  )
  UpperCamelCase_: int   				=    {int(UpperCAmelCase__  ): v for k, v in idalabel.items()}
  UpperCamelCase_: Optional[Any]   				=    idalabel
  UpperCamelCase_: Optional[Any]   				=    {v: k for k, v in idalabel.items()}

  return config

def   snake_case (UpperCAmelCase__  )	->    int:
  UpperCamelCase_: Dict   				=    []

  # stem
  # fmt: off
  rename_keys.append(('backbone.0.body.patch_embed.proj.weight', 'model.backbone.model.embeddings.patch_embeddings.projection.weight')  )
  rename_keys.append(('backbone.0.body.patch_embed.proj.bias', 'model.backbone.model.embeddings.patch_embeddings.projection.bias')  )
  rename_keys.append(('backbone.0.body.patch_embed.norm.weight', 'model.backbone.model.embeddings.norm.weight')  )
  rename_keys.append(('backbone.0.body.patch_embed.norm.bias', 'model.backbone.model.embeddings.norm.bias')  )
  # stages
  for i in range(len(config.backbone_config.depths  )  ):
    for j in range(config.backbone_config.depths[i]  ):
      rename_keys.append((F'''backbone.0.body.layers.{i}.blocks.{j}.norm1.weight''', F'''model.backbone.model.encoder.layers.{i}.blocks.{j}.layernorm_before.weight''')  )
      rename_keys.append((F'''backbone.0.body.layers.{i}.blocks.{j}.norm1.bias''', F'''model.backbone.model.encoder.layers.{i}.blocks.{j}.layernorm_before.bias''')  )
      rename_keys.append((F'''backbone.0.body.layers.{i}.blocks.{j}.attn.relative_position_bias_table''', F'''model.backbone.model.encoder.layers.{i}.blocks.{j}.attention.self.relative_position_bias_table''')  )
      rename_keys.append((F'''backbone.0.body.layers.{i}.blocks.{j}.attn.relative_position_index''', F'''model.backbone.model.encoder.layers.{i}.blocks.{j}.attention.self.relative_position_index''')  )
      rename_keys.append((F'''backbone.0.body.layers.{i}.blocks.{j}.attn.proj.weight''', F'''model.backbone.model.encoder.layers.{i}.blocks.{j}.attention.output.dense.weight''')  )
      rename_keys.append((F'''backbone.0.body.layers.{i}.blocks.{j}.attn.proj.bias''', F'''model.backbone.model.encoder.layers.{i}.blocks.{j}.attention.output.dense.bias''')  )
      rename_keys.append((F'''backbone.0.body.layers.{i}.blocks.{j}.norm2.weight''', F'''model.backbone.model.encoder.layers.{i}.blocks.{j}.layernorm_after.weight''')  )
      rename_keys.append((F'''backbone.0.body.layers.{i}.blocks.{j}.norm2.bias''', F'''model.backbone.model.encoder.layers.{i}.blocks.{j}.layernorm_after.bias''')  )
      rename_keys.append((F'''backbone.0.body.layers.{i}.blocks.{j}.mlp.fc1.weight''', F'''model.backbone.model.encoder.layers.{i}.blocks.{j}.intermediate.dense.weight''')  )
      rename_keys.append((F'''backbone.0.body.layers.{i}.blocks.{j}.mlp.fc1.bias''', F'''model.backbone.model.encoder.layers.{i}.blocks.{j}.intermediate.dense.bias''')  )
      rename_keys.append((F'''backbone.0.body.layers.{i}.blocks.{j}.mlp.fc2.weight''', F'''model.backbone.model.encoder.layers.{i}.blocks.{j}.output.dense.weight''')  )
      rename_keys.append((F'''backbone.0.body.layers.{i}.blocks.{j}.mlp.fc2.bias''', F'''model.backbone.model.encoder.layers.{i}.blocks.{j}.output.dense.bias''')  )

    if i < 3:
      rename_keys.append((F'''backbone.0.body.layers.{i}.downsample.reduction.weight''', F'''model.backbone.model.encoder.layers.{i}.downsample.reduction.weight''')  )
      rename_keys.append((F'''backbone.0.body.layers.{i}.downsample.norm.weight''', F'''model.backbone.model.encoder.layers.{i}.downsample.norm.weight''')  )
      rename_keys.append((F'''backbone.0.body.layers.{i}.downsample.norm.bias''', F'''model.backbone.model.encoder.layers.{i}.downsample.norm.bias''')  )

  rename_keys.append(('backbone.0.body.norm1.weight', 'model.backbone.model.hidden_states_norms.stage2.weight')  )
  rename_keys.append(('backbone.0.body.norm1.bias', 'model.backbone.model.hidden_states_norms.stage2.bias')  )
  rename_keys.append(('backbone.0.body.norm2.weight', 'model.backbone.model.hidden_states_norms.stage3.weight')  )
  rename_keys.append(('backbone.0.body.norm2.bias', 'model.backbone.model.hidden_states_norms.stage3.bias')  )
  rename_keys.append(('backbone.0.body.norm3.weight', 'model.backbone.model.hidden_states_norms.stage4.weight')  )
  rename_keys.append(('backbone.0.body.norm3.bias', 'model.backbone.model.hidden_states_norms.stage4.bias')  )

  # transformer encoder
  for i in range(config.encoder_layers  ):
    rename_keys.append((F'''transformer.encoder.layers.{i}.self_attn.sampling_offsets.weight''', F'''model.encoder.layers.{i}.self_attn.sampling_offsets.weight''')  )
    rename_keys.append((F'''transformer.encoder.layers.{i}.self_attn.sampling_offsets.bias''', F'''model.encoder.layers.{i}.self_attn.sampling_offsets.bias''')  )
    rename_keys.append((F'''transformer.encoder.layers.{i}.self_attn.attention_weights.weight''', F'''model.encoder.layers.{i}.self_attn.attention_weights.weight''')  )
    rename_keys.append((F'''transformer.encoder.layers.{i}.self_attn.attention_weights.bias''', F'''model.encoder.layers.{i}.self_attn.attention_weights.bias''')  )
    rename_keys.append((F'''transformer.encoder.layers.{i}.self_attn.value_proj.weight''', F'''model.encoder.layers.{i}.self_attn.value_proj.weight''')  )
    rename_keys.append((F'''transformer.encoder.layers.{i}.self_attn.value_proj.bias''', F'''model.encoder.layers.{i}.self_attn.value_proj.bias''')  )
    rename_keys.append((F'''transformer.encoder.layers.{i}.self_attn.output_proj.weight''', F'''model.encoder.layers.{i}.self_attn.output_proj.weight''')  )
    rename_keys.append((F'''transformer.encoder.layers.{i}.self_attn.output_proj.bias''', F'''model.encoder.layers.{i}.self_attn.output_proj.bias''')  )
    rename_keys.append((F'''transformer.encoder.layers.{i}.norm1.weight''', F'''model.encoder.layers.{i}.self_attn_layer_norm.weight''')  )
    rename_keys.append((F'''transformer.encoder.layers.{i}.norm1.bias''', F'''model.encoder.layers.{i}.self_attn_layer_norm.bias''')  )
    rename_keys.append((F'''transformer.encoder.layers.{i}.linear1.weight''', F'''model.encoder.layers.{i}.fc1.weight''')  )
    rename_keys.append((F'''transformer.encoder.layers.{i}.linear1.bias''', F'''model.encoder.layers.{i}.fc1.bias''')  )
    rename_keys.append((F'''transformer.encoder.layers.{i}.linear2.weight''', F'''model.encoder.layers.{i}.fc2.weight''')  )
    rename_keys.append((F'''transformer.encoder.layers.{i}.linear2.bias''', F'''model.encoder.layers.{i}.fc2.bias''')  )
    rename_keys.append((F'''transformer.encoder.layers.{i}.norm2.weight''', F'''model.encoder.layers.{i}.final_layer_norm.weight''')  )
    rename_keys.append((F'''transformer.encoder.layers.{i}.norm2.bias''', F'''model.encoder.layers.{i}.final_layer_norm.bias''')  )

  # transformer decoder
  for i in range(config.decoder_layers  ):
    rename_keys.append((F'''transformer.decoder.layers.{i}.cross_attn.sampling_offsets.weight''', F'''model.decoder.layers.{i}.encoder_attn.sampling_offsets.weight''')  )
    rename_keys.append((F'''transformer.decoder.layers.{i}.cross_attn.sampling_offsets.bias''', F'''model.decoder.layers.{i}.encoder_attn.sampling_offsets.bias''')  )
    rename_keys.append((F'''transformer.decoder.layers.{i}.cross_attn.attention_weights.weight''', F'''model.decoder.layers.{i}.encoder_attn.attention_weights.weight''')  )
    rename_keys.append((F'''transformer.decoder.layers.{i}.cross_attn.attention_weights.bias''', F'''model.decoder.layers.{i}.encoder_attn.attention_weights.bias''')  )
    rename_keys.append((F'''transformer.decoder.layers.{i}.cross_attn.value_proj.weight''', F'''model.decoder.layers.{i}.encoder_attn.value_proj.weight''')  )
    rename_keys.append((F'''transformer.decoder.layers.{i}.cross_attn.value_proj.bias''', F'''model.decoder.layers.{i}.encoder_attn.value_proj.bias''')  )
    rename_keys.append((F'''transformer.decoder.layers.{i}.cross_attn.output_proj.weight''', F'''model.decoder.layers.{i}.encoder_attn.output_proj.weight''')  )
    rename_keys.append((F'''transformer.decoder.layers.{i}.cross_attn.output_proj.bias''', F'''model.decoder.layers.{i}.encoder_attn.output_proj.bias''')  )
    rename_keys.append((F'''transformer.decoder.layers.{i}.norm1.weight''', F'''model.decoder.layers.{i}.encoder_attn_layer_norm.weight''')  )
    rename_keys.append((F'''transformer.decoder.layers.{i}.norm1.bias''', F'''model.decoder.layers.{i}.encoder_attn_layer_norm.bias''')  )
    rename_keys.append((F'''transformer.decoder.layers.{i}.self_attn.out_proj.weight''', F'''model.decoder.layers.{i}.self_attn.out_proj.weight''')  )
    rename_keys.append((F'''transformer.decoder.layers.{i}.self_attn.out_proj.bias''', F'''model.decoder.layers.{i}.self_attn.out_proj.bias''')  )
    rename_keys.append((F'''transformer.decoder.layers.{i}.norm2.weight''', F'''model.decoder.layers.{i}.self_attn_layer_norm.weight''')  )
    rename_keys.append((F'''transformer.decoder.layers.{i}.norm2.bias''', F'''model.decoder.layers.{i}.self_attn_layer_norm.bias''')  )
    rename_keys.append((F'''transformer.decoder.layers.{i}.linear1.weight''', F'''model.decoder.layers.{i}.fc1.weight''')  )
    rename_keys.append((F'''transformer.decoder.layers.{i}.linear1.bias''', F'''model.decoder.layers.{i}.fc1.bias''')  )
    rename_keys.append((F'''transformer.decoder.layers.{i}.linear2.weight''', F'''model.decoder.layers.{i}.fc2.weight''')  )
    rename_keys.append((F'''transformer.decoder.layers.{i}.linear2.bias''', F'''model.decoder.layers.{i}.fc2.bias''')  )
    rename_keys.append((F'''transformer.decoder.layers.{i}.norm3.weight''', F'''model.decoder.layers.{i}.final_layer_norm.weight''')  )
    rename_keys.append((F'''transformer.decoder.layers.{i}.norm3.bias''', F'''model.decoder.layers.{i}.final_layer_norm.bias''')  )

  # fmt: on

  return rename_keys

def   snake_case (UpperCAmelCase__     ,   UpperCAmelCase__     ,   UpperCAmelCase__  )	->    List[Any]:
  UpperCamelCase_: str   				=    dct.pop(UpperCAmelCase__  )
  UpperCamelCase_: List[str]   				=    val

def   snake_case (UpperCAmelCase__     ,   UpperCAmelCase__  )	->    List[str]:
  UpperCamelCase_: Tuple   				=    [int(backbone_config.embed_dim * 2**i  ) for i in range(len(backbone_config.depths  )  )]
  for i in range(len(backbone_config.depths  )  ):
    UpperCamelCase_: int   				=    num_features[i]
    for j in range(backbone_config.depths[i]  ):
      # fmt: off
      # read in weights + bias of input projection layer (in original implementation, this is a single matrix + bias)
      UpperCamelCase_: str   				=    state_dict.pop(F'''backbone.0.body.layers.{i}.blocks.{j}.attn.qkv.weight'''  )
      UpperCamelCase_: Dict   				=    state_dict.pop(F'''backbone.0.body.layers.{i}.blocks.{j}.attn.qkv.bias'''  )
      # next, add query, keys and values (in that order) to the state dict
      UpperCamelCase_: Any   				=    in_proj_weight[:dim, :]
      UpperCamelCase_: List[str]   				=    in_proj_bias[: dim]
      UpperCamelCase_: Any   				=    in_proj_weight[
          dim : dim * 2, :
      ]
      UpperCamelCase_: List[str]   				=    in_proj_bias[
          dim : dim * 2
      ]
      UpperCamelCase_: Tuple   				=    in_proj_weight[
          -dim :, :
      ]
      UpperCamelCase_: str   				=    in_proj_bias[-dim :]
      # fmt: on

def   snake_case (UpperCAmelCase__     ,   UpperCAmelCase__  )	->    str:
  # transformer decoder self-attention layers
  UpperCamelCase_: Optional[int]   				=    config.d_model
  for i in range(config.decoder_layers  ):
    # read in weights + bias of input projection layer of self-attention
    UpperCamelCase_: Union[str, Any]   				=    state_dict.pop(F'''transformer.decoder.layers.{i}.self_attn.in_proj_weight'''  )
    UpperCamelCase_: Dict   				=    state_dict.pop(F'''transformer.decoder.layers.{i}.self_attn.in_proj_bias'''  )
    # next, add query, keys and values (in that order) to the state dict
    UpperCamelCase_: Tuple   				=    in_proj_weight[:hidden_size, :]
    UpperCamelCase_: Dict   				=    in_proj_bias[:hidden_size]
    UpperCamelCase_: List[str]   				=    in_proj_weight[
        hidden_size : hidden_size * 2, :
    ]
    UpperCamelCase_: Union[str, Any]   				=    in_proj_bias[hidden_size : hidden_size * 2]
    UpperCamelCase_: Optional[int]   				=    in_proj_weight[-hidden_size:, :]
    UpperCamelCase_: Tuple   				=    in_proj_bias[-hidden_size:]

def   snake_case ()	->    Tuple:
  UpperCamelCase_: int   				=    'http://images.cocodataset.org/val2017/000000039769.jpg'
  UpperCamelCase_: Tuple   				=    Image.open(requests.get(UpperCAmelCase__     ,   stream=UpperCAmelCase__  ).raw  )

  return im

@torch.no_grad()
def   snake_case (UpperCAmelCase__     ,   UpperCAmelCase__     ,   UpperCAmelCase__  )	->    Optional[int]:
  UpperCamelCase_: List[str]   				=    get_deta_config(UpperCAmelCase__  )

  # load original state dict
  if model_name == "deta-swin-large":
    UpperCamelCase_: List[Any]   				=    hf_hub_download(repo_id='nielsr/deta-checkpoints'     ,   filename='adet_swin_ft.pth'  )
  elif model_name == "deta-swin-large-o365":
    UpperCamelCase_: str   				=    hf_hub_download(repo_id='jozhang97/deta-swin-l-o365'     ,   filename='deta_swin_pt_o365.pth'  )
  else:
    raise ValueError(F'''Model name {model_name} not supported'''  )

  UpperCamelCase_: Any   				=    torch.load(UpperCAmelCase__     ,   map_location='cpu'  )['model']

  # original state dict
  for name, param in state_dict.items():
    print(UpperCAmelCase__     ,   param.shape  )

  # rename keys
  UpperCamelCase_: Any   				=    create_rename_keys(UpperCAmelCase__  )
  for src, dest in rename_keys:
    rename_key(UpperCAmelCase__     ,   UpperCAmelCase__     ,   UpperCAmelCase__  )
  read_in_swin_q_k_v(UpperCAmelCase__     ,   config.backbone_config  )
  read_in_decoder_q_k_v(UpperCAmelCase__     ,   UpperCAmelCase__  )

  # fix some prefixes
  for key in state_dict.copy().keys():
    if "transformer.decoder.class_embed" in key or "transformer.decoder.bbox_embed" in key:
      UpperCamelCase_: Dict   				=    state_dict.pop(UpperCAmelCase__  )
      UpperCamelCase_: Any   				=    val
    if "input_proj" in key:
      UpperCamelCase_: Optional[int]   				=    state_dict.pop(UpperCAmelCase__  )
      UpperCamelCase_: Tuple   				=    val
    if "level_embed" in key or "pos_trans" in key or "pix_trans" in key or "enc_output" in key:
      UpperCamelCase_: str   				=    state_dict.pop(UpperCAmelCase__  )
      UpperCamelCase_: Tuple   				=    val

    # finally, create HuggingFace model and load state dict
  UpperCamelCase_: Any   				=    DetaForObjectDetection(UpperCAmelCase__  )
  model.load_state_dict(UpperCAmelCase__  )
  model.eval()

  UpperCamelCase_: int   				=    'cuda' if torch.cuda.is_available() else 'cpu'
  model.to(UpperCAmelCase__  )

  # load image processor
  UpperCamelCase_: Optional[Any]   				=    DetaImageProcessor(format='coco_detection'  )

  # verify our conversion on image
  UpperCamelCase_: Optional[int]   				=    prepare_img()
  UpperCamelCase_: List[Any]   				=    processor(images=UpperCAmelCase__     ,   return_tensors='pt'  )
  UpperCamelCase_: str   				=    encoding['pixel_values']
  UpperCamelCase_: Any   				=    model(pixel_values.to(UpperCAmelCase__  )  )

  # verify logits
  print('Logits:'     ,   outputs.logits[0, :3, :3]  )
  print('Boxes:'     ,   outputs.pred_boxes[0, :3, :3]  )
  if model_name == "deta-swin-large":
    UpperCamelCase_: int   				=    torch.tensor(
        [[-7.6308, -2.8485, -5.3737], [-7.2037, -4.5505, -4.8027], [-7.2943, -4.2611, -4.6617]]  )
    UpperCamelCase_: Any   				=    torch.tensor([[0.4987, 0.4969, 0.9999], [0.2549, 0.5498, 0.4805], [0.5498, 0.2757, 0.0569]]  )
  elif model_name == "deta-swin-large-o365":
    UpperCamelCase_: Dict   				=    torch.tensor(
        [[-8.0122, -3.5720, -4.9717], [-8.1547, -3.6886, -4.6389], [-7.6610, -3.6194, -5.0134]]  )
    UpperCamelCase_: int   				=    torch.tensor([[0.2523, 0.5549, 0.4881], [0.7715, 0.4149, 0.4601], [0.5503, 0.2753, 0.0575]]  )
  assert torch.allclose(outputs.logits[0, :3, :3]     ,   expected_logits.to(UpperCAmelCase__  )     ,   atol=1E-4  )
  assert torch.allclose(outputs.pred_boxes[0, :3, :3]     ,   expected_boxes.to(UpperCAmelCase__  )     ,   atol=1E-4  )
  print('Everything ok!'  )

  if pytorch_dump_folder_path:
    # Save model and processor
    logger.info(F'''Saving PyTorch model and processor to {pytorch_dump_folder_path}...'''  )
    Path(UpperCAmelCase__  ).mkdir(exist_ok=UpperCAmelCase__  )
    model.save_pretrained(UpperCAmelCase__  )
    processor.save_pretrained(UpperCAmelCase__  )

  # Push to hub
  if push_to_hub:
    print('Pushing model and processor to hub...'  )
    model.push_to_hub(F'''jozhang97/{model_name}'''  )
    processor.push_to_hub(F'''jozhang97/{model_name}'''  )


if __name__ == "__main__":
   A_		:		Optional[Any]		     =			argparse.ArgumentParser()

   parser.add_argument(
       '--model_name',
       type=str,
       default='deta-swin-large',
       choices=['deta-swin-large', 'deta-swin-large-o365'],
       help='Name of the model you\'d like to convert.',
   )
   parser.add_argument(
       '--pytorch_dump_folder_path',
       default=None,
       type=str,
       help='Path to the folder to output PyTorch model.',
   )
   parser.add_argument(
       '--push_to_hub', action='store_true', help='Whether or not to push the converted model to the 🤗 hub.'
   )
   A_		:		Optional[int]		     =			parser.parse_args()
   convert_deta_checkpoint(args.model_name, args.pytorch_dump_folder_path, args.push_to_hub)







import os
import sys
import tempfile

import torch

from .state import AcceleratorState
from .utils import PrecisionType, PrepareForLaunch, is_mps_available, patch_environment

def   snake_case (UpperCAmelCase__     ,   UpperCAmelCase__=()     ,   UpperCAmelCase__=None     ,   UpperCAmelCase__="no"     ,   UpperCAmelCase__="29500"  )	->    List[Any]:
  UpperCamelCase_: Any   				=    False
  UpperCamelCase_: List[str]   				=    False
  if any(key.startswith('KAGGLE'  ) for key in os.environ.keys()  ):
    UpperCamelCase_: List[Any]   				=    True
  elif "IPython" in sys.modules:
    UpperCamelCase_: List[Any]   				=    'google.colab' in str(sys.modules['IPython'].get_ipython()  )

  try:
    UpperCamelCase_: Optional[int]   				=    PrecisionType(mixed_precision.lower()  )
  except ValueError:
    raise ValueError(
        F'''Unknown mixed_precision mode: {args.mixed_precision.lower()}. Choose between {PrecisionType.list()}.'''  )

  if (in_colab or in_kaggle) and (os.environ.get('TPU_NAME'     ,   UpperCAmelCase__  ) is not None):
    # TPU launch
    import torch_xla.distributed.xla_multiprocessing as xmp

    if len(AcceleratorState._shared_state  ) > 0:
      raise ValueError(
          'To train on TPU in Colab or Kaggle Kernel, the `Accelerator` should only be initialized inside '
          'your training function. Restart your notebook and make sure no cells initializes an '
          '`Accelerator`.'  )
    if num_processes is None:
      UpperCamelCase_: List[str]   				=    8

    UpperCamelCase_: str   				=    PrepareForLaunch(UpperCAmelCase__     ,   distributed_type='TPU'  )
    print(F'''Launching a training on {num_processes} TPU cores.'''  )
    xmp.spawn(UpperCAmelCase__     ,   args=UpperCAmelCase__     ,   nprocs=UpperCAmelCase__     ,   start_method='fork'  )
  elif in_colab:
    # No need for a distributed launch otherwise as it's either CPU or one GPU.
    if torch.cuda.is_available():
      print('Launching training on one GPU.'  )
    else:
      print('Launching training on one CPU.'  )
    function(*UpperCAmelCase__  )
  else:
    if num_processes is None:
      raise ValueError(
          'You have to specify the number of GPUs you would like to use, add `num_processes=...` to your call.'  )

    if num_processes > 1:
      # Multi-GPU launch
      from torch.multiprocessing import start_processes
      from torch.multiprocessing.spawn import ProcessRaisedException

      if len(AcceleratorState._shared_state  ) > 0:
        raise ValueError(
            'To launch a multi-GPU training from your notebook, the `Accelerator` should only be initialized '
            'inside your training function. Restart your notebook and make sure no cells initializes an '
            '`Accelerator`.'  )

      if torch.cuda.is_initialized():
        raise ValueError(
            'To launch a multi-GPU training from your notebook, you need to avoid running any instruction '
            'using `torch.cuda` in any cell. Restart your notebook and make sure no cells use any CUDA '
            'function.'  )

      # torch.distributed will expect a few environment variable to be here. We set the ones common to each
      # process here (the other ones will be set be the launcher).
      with patch_environment(
          world_size=UpperCAmelCase__     ,   master_addr='127.0.01'     ,   master_port=UpperCAmelCase__     ,   mixed_precision=UpperCAmelCase__  ):
        UpperCamelCase_: str   				=    PrepareForLaunch(UpperCAmelCase__     ,   distributed_type='MULTI_GPU'  )
        print(F'''Launching training on {num_processes} GPUs.'''  )
        try:
          start_processes(UpperCAmelCase__     ,   args=UpperCAmelCase__     ,   nprocs=UpperCAmelCase__     ,   start_method='fork'  )
        except ProcessRaisedException as e:
          if "Cannot re-initialize CUDA in forked subprocess" in e.args[0]:
            raise RuntimeError(
                'CUDA has been initialized before the `notebook_launcher` could create a forked subprocess. '
                'This likely stems from an outside import causing issues once the `notebook_launcher()` is called. '
                'Please review your imports and test them when running the `notebook_launcher()` to identify '
                'which one is problematic.'  ) from e

    else:
      # No need for a distributed launch otherwise as it's either CPU, GPU or MPS.
      if is_mps_available():
        UpperCamelCase_: Tuple   				=    '1'
        print('Launching training on MPS.'  )
      elif torch.cuda.is_available():
        print('Launching training on one GPU.'  )
      else:
        print('Launching training on CPU.'  )
      function(*UpperCAmelCase__  )

def   snake_case (UpperCAmelCase__     ,   UpperCAmelCase__=()     ,   UpperCAmelCase__=2  )	->    Optional[int]:
  from torch.multiprocessing import start_processes

  with tempfile.NamedTemporaryFile() as tmp_file:
    # torch.distributed will expect a few environment variable to be here. We set the ones common to each
    # process here (the other ones will be set be the launcher).
    with patch_environment(
        world_size=UpperCAmelCase__     ,   master_addr='127.0.01'     ,   master_port='29500'     ,   accelerate_mixed_precision='no'     ,   accelerate_debug_rdv_file=tmp_file.name     ,   accelerate_use_cpu='yes'     ,   ):
      UpperCamelCase_: str   				=    PrepareForLaunch(UpperCAmelCase__     ,   debug=UpperCAmelCase__  )
      start_processes(UpperCAmelCase__     ,   args=UpperCAmelCase__     ,   nprocs=UpperCAmelCase__     ,   start_method='fork'  )