





"""simple docstring"""





# Copyright 2022 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import argparse
import os
import subprocess

from packaging.version import Version, parse

from accelerate.commands.config.config_args import default_config_file, load_config_from_file


lowercase__				:int				   =       'Run commands across TPU VMs for initial setup before running `accelerate launch`.'


def       lowerCamelCase_       (   UpperCAmelCase_=None  )						->List[Any]:





  """simple docstring"""


  if subparsers is not None:
    __UpperCAmelCase    :						Union[str, Any]       =						subparsers.add_parser('''tpu-config'''  ,			description=_description  )
  else:
    __UpperCAmelCase    :						Any       =						argparse.ArgumentParser('''Accelerate tpu-config command'''  ,			description=_description  )
  # Core arguments
  __UpperCAmelCase    :						Dict       =						parser.add_argument_group(
      '''Config Arguments'''  ,			'''Arguments that can be configured through `accelerate config`.'''  )
  config_args.add_argument(
      '''--config_file'''  ,			type=UpperCAmelCase_  ,			default=UpperCAmelCase_  ,			help='''Path to the config file to use for accelerate.'''  ,			)
  config_args.add_argument(
      '''--tpu_name'''  ,			default=UpperCAmelCase_  ,			help='''The name of the TPU to use. If not specified, will use the TPU specified in the config file.'''  ,			)
  config_args.add_argument(
      '''--tpu_zone'''  ,			default=UpperCAmelCase_  ,			help='''The zone of the TPU to use. If not specified, will use the zone specified in the config file.'''  ,			)
  __UpperCAmelCase    :						Tuple       =						parser.add_argument_group('''TPU Arguments'''  ,			'''Arguments for options ran inside the TPU.'''  )
  pod_args.add_argument(
      '''--use_alpha'''  ,			action='''store_true'''  ,			help='''Whether to use `gcloud alpha` when running the TPU training script instead of `gcloud`.'''  ,			)
  pod_args.add_argument(
      '''--command_file'''  ,			default=UpperCAmelCase_  ,			help='''The path to the file containing the commands to run on the pod on startup.'''  ,			)
  pod_args.add_argument(
      '''--command'''  ,			action='''append'''  ,			nargs='''+'''  ,			help='''A command to run on the pod. Can be passed multiple times.'''  ,			)
  pod_args.add_argument(
      '''--install_accelerate'''  ,			action='''store_true'''  ,			help='''Whether to install accelerate on the pod. Defaults to False.'''  ,			)
  pod_args.add_argument(
      '''--accelerate_version'''  ,			default='''latest'''  ,			help='''The version of accelerate to install on the pod. If not specified, will use the latest pypi version. Specify \'dev\' to install from GitHub.'''  ,			)
  pod_args.add_argument(
      '''--debug'''  ,			action='''store_true'''  ,			help='''If set, will print the command that would be run instead of running it.'''  )

  if subparsers is not None:
    parser.set_defaults(func=UpperCAmelCase_  )
  return parser


def       lowerCamelCase_       (   UpperCAmelCase_  )						->List[Any]:





  """simple docstring"""


  __UpperCAmelCase    :						Dict       =						None

  # Get the default from the config file if it exists.
  if args.config_file is not None or os.path.isfile(UpperCAmelCase_  ):
    __UpperCAmelCase    :						Union[str, Any]       =						load_config_from_file(args.config_file  )
    if not args.command_file and defaults.command_file is not None and not args.command:
      __UpperCAmelCase    :						int       =						defaults.command_file
    if not args.command and defaults.commands is not None:
      __UpperCAmelCase    :						Optional[Any]       =						defaults.commands
    if not args.tpu_name:
      __UpperCAmelCase    :						List[str]       =						defaults.tpu_name
    if not args.tpu_zone:
      __UpperCAmelCase    :						Dict       =						defaults.tpu_zone
  if args.accelerate_version == "dev":
    __UpperCAmelCase    :						Any       =						'''git+https://github.com/huggingface/accelerate.git'''
  elif args.accelerate_version == "latest":
    __UpperCAmelCase    :						int       =						'''accelerate -U'''
  elif isinstance(parse(args.accelerate_version  )  ,			UpperCAmelCase_  ):
    __UpperCAmelCase    :						List[Any]       =						f'''accelerate=={args.accelerate_version}'''

  if not args.command_file and not args.command:
    raise ValueError('''You must specify either a command file or a command to run on the pod.'''  )

  if args.command_file:
    with open(args.command_file  ,			'''r'''  ) as f:
      __UpperCAmelCase    :						List[str]       =						[f.read().splitlines()]

    # To turn list of lists into list of strings
  if isinstance(args.command[0]  ,			UpperCAmelCase_  ):
    __UpperCAmelCase    :						Optional[int]       =						[line for cmd in args.command for line in cmd]
  # Default to the shared folder and install accelerate
  __UpperCAmelCase    :						int       =						['''cd /usr/share''']
  if args.install_accelerate:
    new_cmd += [f'''pip install {args.accelerate_version}''']
  new_cmd += args.command
  __UpperCAmelCase    :						str       =						'''; '''.join(UpperCAmelCase_  )

  # Then send it to gcloud
  # Eventually try to use google-api-core to do this instead of subprocess
  __UpperCAmelCase    :						Tuple       =						['''gcloud''']
  if args.use_alpha:
    cmd += ["alpha"]
  cmd += [
      "compute",
      "tpus",
      "tpu-vm",
      "ssh",
      args.tpu_name,
      "--zone",
      args.tpu_zone,
      "--command",
      args.command,
      "--worker",
      "all",
  ]
  if args.debug:
    print(f'''Running {' '.join(UpperCAmelCase_  )}'''  )
    return
  subprocess.run(UpperCAmelCase_  )
  print('''Successfully setup pod.'''  )







def       lowerCamelCase_       (   )						->List[Any]:





  """simple docstring"""


  __UpperCAmelCase    :						Optional[int]       =						tpu_command_parser()
  __UpperCAmelCase    :						Optional[Any]       =						parser.parse_args()

  tpu_command_launcher(UpperCAmelCase_  )






"""simple docstring"""





from dataclasses import dataclass
from typing import Dict, Optional, Union

import torch
import torch.nn.functional as F
from torch import nn

from ..configuration_utils import ConfigMixin, register_to_config
from ..utils import BaseOutput
from .attention import BasicTransformerBlock
from .attention_processor import AttentionProcessor, AttnProcessor
from .embeddings import TimestepEmbedding, Timesteps
from .modeling_utils import ModelMixin




@dataclass
class 							snake_case							(			__UpperCAmelCase ):

      '''simple docstring'''



      _A     :       torch.FloatTensor




class 							snake_case							(			__UpperCAmelCase      ,  __UpperCAmelCase ):

      '''simple docstring'''



      @register_to_config
      def __init__(							self       :      Union[str, Any]						,				__lowercase       :      int = 32						,				__lowercase       :      int = 64						,				__lowercase       :      int = 20						,				__lowercase       :      int = 768						,				__lowercase       :      Optional[int]=77						,				__lowercase       :      Union[str, Any]=4						,				__lowercase       :      float = 0.0						,				__lowercase       :      str = "silu"						,				__lowercase       :      Optional[str] = None						,				__lowercase       :      Optional[str] = None						,				__lowercase       :      Optional[str] = "linear"						,				__lowercase       :      Optional[str] = "prd"						,				__lowercase       :      Optional[int] = None						,				__lowercase       :      Optional[int] = None						,				__lowercase       :      Optional[int] = None						,				):


        '''simple docstring'''




        super().__init__()
        __UpperCAmelCase    :						int       =						num_attention_heads
        __UpperCAmelCase    :						List[Any]       =						attention_head_dim
        __UpperCAmelCase    :						int       =						num_attention_heads * attention_head_dim
        __UpperCAmelCase    :						List[str]       =						additional_embeddings

        __UpperCAmelCase    :						Optional[int]       =						time_embed_dim or inner_dim
        __UpperCAmelCase    :						Tuple       =						embedding_proj_dim or embedding_dim
        __UpperCAmelCase    :						Dict       =						clip_embed_dim or embedding_dim

        __UpperCAmelCase    :						Dict       =						Timesteps(__lowercase						,				__lowercase						,				0 )
        __UpperCAmelCase    :						List[str]       =						TimestepEmbedding(__lowercase						,				__lowercase						,				out_dim=__lowercase						,				act_fn=__lowercase )

        __UpperCAmelCase    :						Any       =						nn.Linear(__lowercase						,				__lowercase )

        if embedding_proj_norm_type is None:
          __UpperCAmelCase    :						Dict       =						None
        elif embedding_proj_norm_type == "layer":
          __UpperCAmelCase    :						Any       =						nn.LayerNorm(__lowercase )
        else:
          raise ValueError(f'''unsupported embedding_proj_norm_type: {embedding_proj_norm_type}''' )

        __UpperCAmelCase    :						List[Any]       =						nn.Linear(__lowercase						,				__lowercase )

        if encoder_hid_proj_type is None:
          __UpperCAmelCase    :						int       =						None
        elif encoder_hid_proj_type == "linear":
          __UpperCAmelCase    :						int       =						nn.Linear(__lowercase						,				__lowercase )
        else:
          raise ValueError(f'''unsupported encoder_hid_proj_type: {encoder_hid_proj_type}''' )

        __UpperCAmelCase    :						str       =						nn.Parameter(torch.zeros(1						,				num_embeddings + additional_embeddings						,				__lowercase ) )

        if added_emb_type == "prd":
          __UpperCAmelCase    :						Optional[int]       =						nn.Parameter(torch.zeros(1						,				1						,				__lowercase ) )
        elif added_emb_type is None:
          __UpperCAmelCase    :						str       =						None
        else:
          raise ValueError(
              f'''`added_emb_type`: {added_emb_type} is not supported. Make sure to choose one of `\'prd\'` or `None`.''' )

        __UpperCAmelCase    :						Union[str, Any]       =						nn.ModuleList(
            [
                BasicTransformerBlock(
                    __lowercase						,				__lowercase						,				__lowercase						,				dropout=__lowercase						,				activation_fn='''gelu'''						,				attention_bias=__lowercase						,				)
                for d in range(__lowercase )
            ] )

        if norm_in_type == "layer":
          __UpperCAmelCase    :						Optional[Any]       =						nn.LayerNorm(__lowercase )
        elif norm_in_type is None:
          __UpperCAmelCase    :						List[Any]       =						None
        else:
          raise ValueError(f'''Unsupported norm_in_type: {norm_in_type}.''' )

        __UpperCAmelCase    :						str       =						nn.LayerNorm(__lowercase )

        __UpperCAmelCase    :						List[Any]       =						nn.Linear(__lowercase						,				__lowercase )

        __UpperCAmelCase    :						List[Any]       =						torch.full(
            [num_embeddings + additional_embeddings, num_embeddings + additional_embeddings]						,				-1_0_0_0_0.0 )
        causal_attention_mask.triu_(1 )
        __UpperCAmelCase    :						Optional[Any]       =						causal_attention_mask[None, ...]
        self.register_buffer('''causal_attention_mask'''						,				__lowercase						,				persistent=__lowercase )

        __UpperCAmelCase    :						Any       =						nn.Parameter(torch.zeros(1						,				__lowercase ) )
        __UpperCAmelCase    :						List[str]       =						nn.Parameter(torch.zeros(1						,				__lowercase ) )

      @property
      # Copied from diffusers.models.unet_2d_condition.UNet2DConditionModel.attn_processors
      def       A_					(							self       :      Optional[int] ):


        '''simple docstring'''




        __UpperCAmelCase    :						Optional[int]       =						{}

        def fn_recursive_add_processors(__lowercase       :      str						,				__lowercase       :      torch.nn.Module						,				__lowercase       :      Dict[str, AttentionProcessor] ):
          if hasattr(__lowercase						,				'''set_processor''' ):
            __UpperCAmelCase    :						Optional[Any]       =						module.processor

          for sub_name, child in module.named_children():
            fn_recursive_add_processors(f'''{name}.{sub_name}'''						,				__lowercase						,				__lowercase )

          return processors

        for name, module in self.named_children():
          fn_recursive_add_processors(__lowercase						,				__lowercase						,				__lowercase )

        return processors

      def       A_					(							self       :      Any						,				__lowercase       :      Union[AttentionProcessor, Dict[str, AttentionProcessor]] ):


        '''simple docstring'''




        __UpperCAmelCase    :						Tuple       =						len(self.attn_processors.keys() )

        if isinstance(__lowercase						,				__lowercase ) and len(__lowercase ) != count:
          raise ValueError(
              f'''A dict of processors was passed, but the number of processors {len(__lowercase )} does not match the'''
              f''' number of attention layers: {count}. Please make sure to pass {count} processor classes.''' )

        def fn_recursive_attn_processor(__lowercase       :      str						,				__lowercase       :      torch.nn.Module						,				__lowercase       :      int ):
          if hasattr(__lowercase						,				'''set_processor''' ):
            if not isinstance(__lowercase						,				__lowercase ):
              module.set_processor(__lowercase )
            else:
              module.set_processor(processor.pop(f'''{name}.processor''' ) )

          for sub_name, child in module.named_children():
            fn_recursive_attn_processor(f'''{name}.{sub_name}'''						,				__lowercase						,				__lowercase )

        for name, module in self.named_children():
          fn_recursive_attn_processor(__lowercase						,				__lowercase						,				__lowercase )

      def       A_					(							self       :      Optional[Any] ):


        '''simple docstring'''




        self.set_attn_processor(AttnProcessor() )

      def       A_					(							self       :      Optional[int]						,				__lowercase       :      Dict						,				__lowercase       :      Union[torch.Tensor, float, int]						,				__lowercase       :      torch.FloatTensor						,				__lowercase       :      Optional[torch.FloatTensor] = None						,				__lowercase       :      Optional[torch.BoolTensor] = None						,				__lowercase       :      bool = True						,				):


        '''simple docstring'''




        __UpperCAmelCase    :						List[Any]       =						hidden_states.shape[0]

        __UpperCAmelCase    :						Any       =						timestep
        if not torch.is_tensor(__lowercase ):
          __UpperCAmelCase    :						Union[str, Any]       =						torch.tensor([timesteps]						,				dtype=torch.long						,				device=hidden_states.device )
        elif torch.is_tensor(__lowercase ) and len(timesteps.shape ) == 0:
          __UpperCAmelCase    :						Union[str, Any]       =						timesteps[None].to(hidden_states.device )

        # broadcast to batch dimension in a way that's compatible with ONNX/Core ML
        __UpperCAmelCase    :						List[str]       =						timesteps * torch.ones(__lowercase						,				dtype=timesteps.dtype						,				device=timesteps.device )

        __UpperCAmelCase    :						List[str]       =						self.time_proj(__lowercase )

        # timesteps does not contain any weights and will always return f32 tensors
        # but time_embedding might be fp16, so we need to cast here.
        __UpperCAmelCase    :						Any       =						timesteps_projected.to(dtype=self.dtype )
        __UpperCAmelCase    :						List[str]       =						self.time_embedding(__lowercase )

        if self.embedding_proj_norm is not None:
          __UpperCAmelCase    :						Dict       =						self.embedding_proj_norm(__lowercase )

        __UpperCAmelCase    :						Optional[int]       =						self.embedding_proj(__lowercase )
        if self.encoder_hidden_states_proj is not None and encoder_hidden_states is not None:
          __UpperCAmelCase    :						Dict       =						self.encoder_hidden_states_proj(__lowercase )
        elif self.encoder_hidden_states_proj is not None and encoder_hidden_states is None:
          raise ValueError('''`encoder_hidden_states_proj` requires `encoder_hidden_states` to be set''' )

        __UpperCAmelCase    :						int       =						self.proj_in(__lowercase )

        __UpperCAmelCase    :						Tuple       =						self.positional_embedding.to(hidden_states.dtype )

        __UpperCAmelCase    :						Optional[Any]       =						[]
        __UpperCAmelCase    :						str       =						0

        if encoder_hidden_states is not None:
          additional_embeds.append(__lowercase )
          additional_embeddings_len += encoder_hidden_states.shape[1]

        if len(proj_embeddings.shape ) == 2:
          __UpperCAmelCase    :						str       =						proj_embeddings[:, None, :]

        if len(hidden_states.shape ) == 2:
          __UpperCAmelCase    :						Optional[int]       =						hidden_states[:, None, :]

        __UpperCAmelCase    :						str       =						additional_embeds + [
            proj_embeddings,
            time_embeddings[:, None, :],
            hidden_states,
        ]

        if self.prd_embedding is not None:
          __UpperCAmelCase    :						Dict       =						self.prd_embedding.to(hidden_states.dtype ).expand(__lowercase						,				-1						,				-1 )
          additional_embeds.append(__lowercase )

        __UpperCAmelCase    :						Optional[Any]       =						torch.cat(
            __lowercase						,				dim=1						,				)

        # Allow positional_embedding to not include the `addtional_embeddings` and instead pad it with zeros for these additional tokens
        __UpperCAmelCase    :						Optional[int]       =						additional_embeddings_len + proj_embeddings.shape[1] + 1
        if positional_embeddings.shape[1] < hidden_states.shape[1]:
          __UpperCAmelCase    :						List[str]       =						F.pad(
              __lowercase						,				(
                  0,
                  0,
                  additional_embeddings_len,
                  self.prd_embedding.shape[1] if self.prd_embedding is not None else 0,
              )						,				value=0.0						,				)

        __UpperCAmelCase    :						Dict       =						hidden_states + positional_embeddings

        if attention_mask is not None:
          __UpperCAmelCase    :						Optional[Any]       =						(1 - attention_mask.to(hidden_states.dtype )) * -1_0_0_0_0.0
          __UpperCAmelCase    :						Optional[Any]       =						F.pad(__lowercase						,				(0, self.additional_embeddings)						,				value=0.0 )
          __UpperCAmelCase    :						Dict       =						(attention_mask[:, None, :] + self.causal_attention_mask).to(hidden_states.dtype )
          __UpperCAmelCase    :						Tuple       =						attention_mask.repeat_interleave(self.config.num_attention_heads						,				dim=0 )

        if self.norm_in is not None:
          __UpperCAmelCase    :						str       =						self.norm_in(__lowercase )

        for block in self.transformer_blocks:
          __UpperCAmelCase    :						Dict       =						block(__lowercase						,				attention_mask=__lowercase )

        __UpperCAmelCase    :						Any       =						self.norm_out(__lowercase )

        if self.prd_embedding is not None:
          __UpperCAmelCase    :						List[Any]       =						hidden_states[:, -1]
        else:
          __UpperCAmelCase    :						List[str]       =						hidden_states[:, additional_embeddings_len:]

        __UpperCAmelCase    :						Optional[int]       =						self.proj_to_clip_embeddings(__lowercase )

        if not return_dict:
          return (predicted_image_embedding,)

        return PriorTransformerOutput(predicted_image_embedding=__lowercase )







      def       A_					(							self       :      List[Any]						,				__lowercase       :      Optional[int] ):


        '''simple docstring'''




        __UpperCAmelCase    :						List[str]       =						(prior_latents * self.clip_std) + self.clip_mean
        return prior_latents