





"""simple docstring"""





import os
from shutil import copyfile
from typing import List, Optional, Tuple

from tokenizers import processors

from ...tokenization_utils import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
  from .tokenization_nllb import NllbTokenizer
else:
  lowercase__				:Optional[Any]				   =       None


lowercase__				:List[str]				   =       logging.get_logger(__name__)


lowercase__				:Any				   =       {'vocab_file': 'sentencepiece.bpe.model', 'tokenizer_file': 'tokenizer.json'}

lowercase__				:List[str]				   =       {
    'vocab_file': {
        'facebook/nllb-200-distilled-600M': (
            'https://huggingface.co/facebook/nllb-200-distilled-600M/resolve/main/sentencepiece.bpe.model'
        ),
    },
    'tokenizer_file': {
        'facebook/nllb-200-distilled-600M': (
            'https://huggingface.co/facebook/nllb-200-distilled-600M/resolve/main/tokenizer.json'
        ),
    },
}

lowercase__				:Dict				   =       {
    'facebook/nllb-large-en-ro': 1_0_2_4,
    'facebook/nllb-200-distilled-600M': 1_0_2_4,
}

# fmt: off
lowercase__				:Dict				   =       ['ace_Arab', 'ace_Latn', 'acm_Arab', 'acq_Arab', 'aeb_Arab', 'afr_Latn', 'ajp_Arab', 'aka_Latn', 'amh_Ethi', 'apc_Arab', 'arb_Arab', 'ars_Arab', 'ary_Arab', 'arz_Arab', 'asm_Beng', 'ast_Latn', 'awa_Deva', 'ayr_Latn', 'azb_Arab', 'azj_Latn', 'bak_Cyrl', 'bam_Latn', 'ban_Latn', 'bel_Cyrl', 'bem_Latn', 'ben_Beng', 'bho_Deva', 'bjn_Arab', 'bjn_Latn', 'bod_Tibt', 'bos_Latn', 'bug_Latn', 'bul_Cyrl', 'cat_Latn', 'ceb_Latn', 'ces_Latn', 'cjk_Latn', 'ckb_Arab', 'crh_Latn', 'cym_Latn', 'dan_Latn', 'deu_Latn', 'dik_Latn', 'dyu_Latn', 'dzo_Tibt', 'ell_Grek', 'eng_Latn', 'epo_Latn', 'est_Latn', 'eus_Latn', 'ewe_Latn', 'fao_Latn', 'pes_Arab', 'fij_Latn', 'fin_Latn', 'fon_Latn', 'fra_Latn', 'fur_Latn', 'fuv_Latn', 'gla_Latn', 'gle_Latn', 'glg_Latn', 'grn_Latn', 'guj_Gujr', 'hat_Latn', 'hau_Latn', 'heb_Hebr', 'hin_Deva', 'hne_Deva', 'hrv_Latn', 'hun_Latn', 'hye_Armn', 'ibo_Latn', 'ilo_Latn', 'ind_Latn', 'isl_Latn', 'ita_Latn', 'jav_Latn', 'jpn_Jpan', 'kab_Latn', 'kac_Latn', 'kam_Latn', 'kan_Knda', 'kas_Arab', 'kas_Deva', 'kat_Geor', 'knc_Arab', 'knc_Latn', 'kaz_Cyrl', 'kbp_Latn', 'kea_Latn', 'khm_Khmr', 'kik_Latn', 'kin_Latn', 'kir_Cyrl', 'kmb_Latn', 'kon_Latn', 'kor_Hang', 'kmr_Latn', 'lao_Laoo', 'lvs_Latn', 'lij_Latn', 'lim_Latn', 'lin_Latn', 'lit_Latn', 'lmo_Latn', 'ltg_Latn', 'ltz_Latn', 'lua_Latn', 'lug_Latn', 'luo_Latn', 'lus_Latn', 'mag_Deva', 'mai_Deva', 'mal_Mlym', 'mar_Deva', 'min_Latn', 'mkd_Cyrl', 'plt_Latn', 'mlt_Latn', 'mni_Beng', 'khk_Cyrl', 'mos_Latn', 'mri_Latn', 'zsm_Latn', 'mya_Mymr', 'nld_Latn', 'nno_Latn', 'nob_Latn', 'npi_Deva', 'nso_Latn', 'nus_Latn', 'nya_Latn', 'oci_Latn', 'gaz_Latn', 'ory_Orya', 'pag_Latn', 'pan_Guru', 'pap_Latn', 'pol_Latn', 'por_Latn', 'prs_Arab', 'pbt_Arab', 'quy_Latn', 'ron_Latn', 'run_Latn', 'rus_Cyrl', 'sag_Latn', 'san_Deva', 'sat_Beng', 'scn_Latn', 'shn_Mymr', 'sin_Sinh', 'slk_Latn', 'slv_Latn', 'smo_Latn', 'sna_Latn', 'snd_Arab', 'som_Latn', 'sot_Latn', 'spa_Latn', 'als_Latn', 'srd_Latn', 'srp_Cyrl', 'ssw_Latn', 'sun_Latn', 'swe_Latn', 'swh_Latn', 'szl_Latn', 'tam_Taml', 'tat_Cyrl', 'tel_Telu', 'tgk_Cyrl', 'tgl_Latn', 'tha_Thai', 'tir_Ethi', 'taq_Latn', 'taq_Tfng', 'tpi_Latn', 'tsn_Latn', 'tso_Latn', 'tuk_Latn', 'tum_Latn', 'tur_Latn', 'twi_Latn', 'tzm_Tfng', 'uig_Arab', 'ukr_Cyrl', 'umb_Latn', 'urd_Arab', 'uzn_Latn', 'vec_Latn', 'vie_Latn', 'war_Latn', 'wol_Latn', 'xho_Latn', 'ydd_Hebr', 'yor_Latn', 'yue_Hant', 'zho_Hans', 'zho_Hant', 'zul_Latn']




class 							snake_case							(			__UpperCAmelCase ):

      '''simple docstring'''



      _A     :       Tuple         =       VOCAB_FILES_NAMES
      _A     :       List[str]         =       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      _A     :       List[str]         =       PRETRAINED_VOCAB_FILES_MAP
      _A     :       Optional[int]         =       ['input_ids', 'attention_mask']
      _A     :       int         =       NllbTokenizer

      _A     :       List[int]         =       []
      _A     :       List[int]         =       []

      def __init__(							self       :      Union[str, Any]						,				__lowercase       :      Tuple=None						,				__lowercase       :      List[str]=None						,				__lowercase       :      Union[str, Any]="<s>"						,				__lowercase       :      Union[str, Any]="</s>"						,				__lowercase       :      Dict="</s>"						,				__lowercase       :      Optional[Any]="<s>"						,				__lowercase       :      Union[str, Any]="<unk>"						,				__lowercase       :      Optional[Any]="<pad>"						,				__lowercase       :      int="<mask>"						,				__lowercase       :      Any=None						,				__lowercase       :      int=None						,				__lowercase       :      Dict=None						,				__lowercase       :      Tuple=False						,				**__lowercase       :      int						,				):


        '''simple docstring'''




        __UpperCAmelCase    :						List[Any]       =						AddedToken(__lowercase						,				lstrip=__lowercase						,				rstrip=__lowercase ) if isinstance(__lowercase						,				__lowercase ) else mask_token
        __UpperCAmelCase    :						Any       =						legacy_behaviour
        super().__init__(
            vocab_file=__lowercase						,				tokenizer_file=__lowercase						,				bos_token=__lowercase						,				eos_token=__lowercase						,				sep_token=__lowercase						,				cls_token=__lowercase						,				unk_token=__lowercase						,				pad_token=__lowercase						,				mask_token=__lowercase						,				src_lang=__lowercase						,				tgt_lang=__lowercase						,				additional_special_tokens=__lowercase						,				legacy_behaviour=__lowercase						,				**__lowercase						,				)

        __UpperCAmelCase    :						Union[str, Any]       =						vocab_file
        __UpperCAmelCase    :						List[str]       =						False if not self.vocab_file else True

        __UpperCAmelCase    :						List[Any]       =						FAIRSEQ_LANGUAGE_CODES.copy()

        if additional_special_tokens is not None:
          # Only add those special tokens if they are not already there.
          _additional_special_tokens.extend(
              [t for t in additional_special_tokens if t not in _additional_special_tokens] )

        self.add_special_tokens({'''additional_special_tokens''': _additional_special_tokens} )
        __UpperCAmelCase    :						Tuple       =						{
            lang_code: self.convert_tokens_to_ids(__lowercase ) for lang_code in FAIRSEQ_LANGUAGE_CODES
        }

        __UpperCAmelCase    :						str       =						src_lang if src_lang is not None else '''eng_Latn'''
        __UpperCAmelCase    :						Union[str, Any]       =						self.convert_tokens_to_ids(self._src_lang )
        __UpperCAmelCase    :						Optional[Any]       =						tgt_lang
        self.set_src_lang_special_tokens(self._src_lang )

      @property
      def       A_					(							self       :      Optional[Any] ):


        '''simple docstring'''




        return self._src_lang

      @src_lang.setter
      def       A_					(							self       :      str						,				__lowercase       :      str ):


        '''simple docstring'''




        __UpperCAmelCase    :						List[Any]       =						new_src_lang
        self.set_src_lang_special_tokens(self._src_lang )

      def       A_					(							self       :      Any						,				__lowercase       :      List[int]						,				__lowercase       :      Optional[List[int]] = None ):


        '''simple docstring'''




        if token_ids_a is None:
          return self.prefix_tokens + token_ids_a + self.suffix_tokens
        # We don't expect to process pairs, but leave the pair logic for API consistency
        return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens

      def       A_					(							self       :      str						,				__lowercase       :      List[int]						,				__lowercase       :      Optional[List[int]] = None ):


        '''simple docstring'''




        __UpperCAmelCase    :						Dict       =						[self.sep_token_id]
        __UpperCAmelCase    :						Dict       =						[self.cls_token_id]

        if token_ids_a is None:
          return len(cls + token_ids_a + sep ) * [0]
        return len(cls + token_ids_a + sep + sep + token_ids_a + sep ) * [0]

      def       A_					(							self       :      Optional[Any]						,				__lowercase       :      Union[str, Any]						,				__lowercase       :      str						,				__lowercase       :      Optional[str]						,				__lowercase       :      Optional[str]						,				**__lowercase       :      int ):


        '''simple docstring'''




        if src_lang is None or tgt_lang is None:
          raise ValueError('''Translation requires a `src_lang` and a `tgt_lang` for this model''' )
        __UpperCAmelCase    :						str       =						src_lang
        __UpperCAmelCase    :						Tuple       =						self(__lowercase						,				add_special_tokens=__lowercase						,				return_tensors=__lowercase						,				**__lowercase )
        __UpperCAmelCase    :						Optional[int]       =						self.convert_tokens_to_ids(__lowercase )
        __UpperCAmelCase    :						Any       =						tgt_lang_id
        return inputs

      def       A_					(							self       :      int						,				__lowercase       :      List[str]						,				__lowercase       :      str = "eng_Latn"						,				__lowercase       :      Optional[List[str]] = None						,				__lowercase       :      str = "fra_Latn"						,				**__lowercase       :      Dict						,				):


        '''simple docstring'''




        __UpperCAmelCase    :						str       =						src_lang
        __UpperCAmelCase    :						str       =						tgt_lang
        return super().prepare_seqaseq_batch(__lowercase						,				__lowercase						,				**__lowercase )

      def       A_					(							self       :      int ):


        '''simple docstring'''




        return self.set_src_lang_special_tokens(self.src_lang )

      def       A_					(							self       :      List[Any] ):


        '''simple docstring'''




        return self.set_tgt_lang_special_tokens(self.tgt_lang )

      def       A_					(							self       :      List[Any]						,				__lowercase       :      Dict ):


        '''simple docstring'''




        __UpperCAmelCase    :						Union[str, Any]       =						self.convert_tokens_to_ids(__lowercase )

        if self.legacy_behaviour:
          __UpperCAmelCase    :						int       =						[]
          __UpperCAmelCase    :						Dict       =						[self.eos_token_id, self.cur_lang_code]
        else:
          __UpperCAmelCase    :						Any       =						[self.cur_lang_code]
          __UpperCAmelCase    :						Union[str, Any]       =						[self.eos_token_id]

        __UpperCAmelCase    :						List[Any]       =						self.convert_ids_to_tokens(self.prefix_tokens )
        __UpperCAmelCase    :						List[Any]       =						self.convert_ids_to_tokens(self.suffix_tokens )

        __UpperCAmelCase    :						Union[str, Any]       =						processors.TemplateProcessing(
            single=prefix_tokens_str + ['''$A'''] + suffix_tokens_str						,				pair=prefix_tokens_str + ['''$A''', '''$B'''] + suffix_tokens_str						,				special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str						,				self.prefix_tokens + self.suffix_tokens ) )						,				)

      def       A_					(							self       :      List[str]						,				__lowercase       :      str ):


        '''simple docstring'''




        __UpperCAmelCase    :						Optional[Any]       =						self.convert_tokens_to_ids(__lowercase )
        if self.legacy_behaviour:
          __UpperCAmelCase    :						int       =						[]
          __UpperCAmelCase    :						Dict       =						[self.eos_token_id, self.cur_lang_code]
        else:
          __UpperCAmelCase    :						str       =						[self.cur_lang_code]
          __UpperCAmelCase    :						int       =						[self.eos_token_id]

        __UpperCAmelCase    :						Any       =						self.convert_ids_to_tokens(self.prefix_tokens )
        __UpperCAmelCase    :						Union[str, Any]       =						self.convert_ids_to_tokens(self.suffix_tokens )

        __UpperCAmelCase    :						Optional[int]       =						processors.TemplateProcessing(
            single=prefix_tokens_str + ['''$A'''] + suffix_tokens_str						,				pair=prefix_tokens_str + ['''$A''', '''$B'''] + suffix_tokens_str						,				special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str						,				self.prefix_tokens + self.suffix_tokens ) )						,				)







      def       A_					(							self       :      Union[str, Any]						,				__lowercase       :      str						,				__lowercase       :      Optional[str] = None ):


        '''simple docstring'''




        if not self.can_save_slow_tokenizer:
          raise ValueError(
              '''Your fast tokenizer does not have the necessary information to save the vocabulary for a slow '''
              '''tokenizer.''' )

        if not os.path.isdir(__lowercase ):
          logger.error(f'''Vocabulary path ({save_directory}) should be a directory.''' )
          return
        __UpperCAmelCase    :						Tuple       =						os.path.join(
            __lowercase						,				(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file'''] )

        if os.path.abspath(self.vocab_file ) != os.path.abspath(__lowercase ):
          copyfile(self.vocab_file						,				__lowercase )

        return (out_vocab_file,)






"""simple docstring"""





from dataclasses import dataclass
from typing import Dict, Optional, Union

import torch
import torch.nn.functional as F
from torch import nn

from ..configuration_utils import ConfigMixin, register_to_config
from ..utils import BaseOutput
from .attention import BasicTransformerBlock
from .attention_processor import AttentionProcessor, AttnProcessor
from .embeddings import TimestepEmbedding, Timesteps
from .modeling_utils import ModelMixin




@dataclass
class 							snake_case							(			__UpperCAmelCase ):

      '''simple docstring'''



      _A     :       torch.FloatTensor




class 							snake_case							(			__UpperCAmelCase      ,  __UpperCAmelCase ):

      '''simple docstring'''



      @register_to_config
      def __init__(							self       :      Union[str, Any]						,				__lowercase       :      int = 32						,				__lowercase       :      int = 64						,				__lowercase       :      int = 20						,				__lowercase       :      int = 768						,				__lowercase       :      Optional[int]=77						,				__lowercase       :      Union[str, Any]=4						,				__lowercase       :      float = 0.0						,				__lowercase       :      str = "silu"						,				__lowercase       :      Optional[str] = None						,				__lowercase       :      Optional[str] = None						,				__lowercase       :      Optional[str] = "linear"						,				__lowercase       :      Optional[str] = "prd"						,				__lowercase       :      Optional[int] = None						,				__lowercase       :      Optional[int] = None						,				__lowercase       :      Optional[int] = None						,				):


        '''simple docstring'''




        super().__init__()
        __UpperCAmelCase    :						int       =						num_attention_heads
        __UpperCAmelCase    :						List[Any]       =						attention_head_dim
        __UpperCAmelCase    :						int       =						num_attention_heads * attention_head_dim
        __UpperCAmelCase    :						List[str]       =						additional_embeddings

        __UpperCAmelCase    :						Optional[int]       =						time_embed_dim or inner_dim
        __UpperCAmelCase    :						Tuple       =						embedding_proj_dim or embedding_dim
        __UpperCAmelCase    :						Dict       =						clip_embed_dim or embedding_dim

        __UpperCAmelCase    :						Dict       =						Timesteps(__lowercase						,				__lowercase						,				0 )
        __UpperCAmelCase    :						List[str]       =						TimestepEmbedding(__lowercase						,				__lowercase						,				out_dim=__lowercase						,				act_fn=__lowercase )

        __UpperCAmelCase    :						Any       =						nn.Linear(__lowercase						,				__lowercase )

        if embedding_proj_norm_type is None:
          __UpperCAmelCase    :						Dict       =						None
        elif embedding_proj_norm_type == "layer":
          __UpperCAmelCase    :						Any       =						nn.LayerNorm(__lowercase )
        else:
          raise ValueError(f'''unsupported embedding_proj_norm_type: {embedding_proj_norm_type}''' )

        __UpperCAmelCase    :						List[Any]       =						nn.Linear(__lowercase						,				__lowercase )

        if encoder_hid_proj_type is None:
          __UpperCAmelCase    :						int       =						None
        elif encoder_hid_proj_type == "linear":
          __UpperCAmelCase    :						int       =						nn.Linear(__lowercase						,				__lowercase )
        else:
          raise ValueError(f'''unsupported encoder_hid_proj_type: {encoder_hid_proj_type}''' )

        __UpperCAmelCase    :						str       =						nn.Parameter(torch.zeros(1						,				num_embeddings + additional_embeddings						,				__lowercase ) )

        if added_emb_type == "prd":
          __UpperCAmelCase    :						Optional[int]       =						nn.Parameter(torch.zeros(1						,				1						,				__lowercase ) )
        elif added_emb_type is None:
          __UpperCAmelCase    :						str       =						None
        else:
          raise ValueError(
              f'''`added_emb_type`: {added_emb_type} is not supported. Make sure to choose one of `\'prd\'` or `None`.''' )

        __UpperCAmelCase    :						Union[str, Any]       =						nn.ModuleList(
            [
                BasicTransformerBlock(
                    __lowercase						,				__lowercase						,				__lowercase						,				dropout=__lowercase						,				activation_fn='''gelu'''						,				attention_bias=__lowercase						,				)
                for d in range(__lowercase )
            ] )

        if norm_in_type == "layer":
          __UpperCAmelCase    :						Optional[Any]       =						nn.LayerNorm(__lowercase )
        elif norm_in_type is None:
          __UpperCAmelCase    :						List[Any]       =						None
        else:
          raise ValueError(f'''Unsupported norm_in_type: {norm_in_type}.''' )

        __UpperCAmelCase    :						str       =						nn.LayerNorm(__lowercase )

        __UpperCAmelCase    :						List[Any]       =						nn.Linear(__lowercase						,				__lowercase )

        __UpperCAmelCase    :						List[Any]       =						torch.full(
            [num_embeddings + additional_embeddings, num_embeddings + additional_embeddings]						,				-1_0_0_0_0.0 )
        causal_attention_mask.triu_(1 )
        __UpperCAmelCase    :						Optional[Any]       =						causal_attention_mask[None, ...]
        self.register_buffer('''causal_attention_mask'''						,				__lowercase						,				persistent=__lowercase )

        __UpperCAmelCase    :						Any       =						nn.Parameter(torch.zeros(1						,				__lowercase ) )
        __UpperCAmelCase    :						List[str]       =						nn.Parameter(torch.zeros(1						,				__lowercase ) )

      @property
      # Copied from diffusers.models.unet_2d_condition.UNet2DConditionModel.attn_processors
      def       A_					(							self       :      Optional[int] ):


        '''simple docstring'''




        __UpperCAmelCase    :						Optional[int]       =						{}

        def fn_recursive_add_processors(__lowercase       :      str						,				__lowercase       :      torch.nn.Module						,				__lowercase       :      Dict[str, AttentionProcessor] ):
          if hasattr(__lowercase						,				'''set_processor''' ):
            __UpperCAmelCase    :						Optional[Any]       =						module.processor

          for sub_name, child in module.named_children():
            fn_recursive_add_processors(f'''{name}.{sub_name}'''						,				__lowercase						,				__lowercase )

          return processors

        for name, module in self.named_children():
          fn_recursive_add_processors(__lowercase						,				__lowercase						,				__lowercase )

        return processors

      def       A_					(							self       :      Any						,				__lowercase       :      Union[AttentionProcessor, Dict[str, AttentionProcessor]] ):


        '''simple docstring'''




        __UpperCAmelCase    :						Tuple       =						len(self.attn_processors.keys() )

        if isinstance(__lowercase						,				__lowercase ) and len(__lowercase ) != count:
          raise ValueError(
              f'''A dict of processors was passed, but the number of processors {len(__lowercase )} does not match the'''
              f''' number of attention layers: {count}. Please make sure to pass {count} processor classes.''' )

        def fn_recursive_attn_processor(__lowercase       :      str						,				__lowercase       :      torch.nn.Module						,				__lowercase       :      int ):
          if hasattr(__lowercase						,				'''set_processor''' ):
            if not isinstance(__lowercase						,				__lowercase ):
              module.set_processor(__lowercase )
            else:
              module.set_processor(processor.pop(f'''{name}.processor''' ) )

          for sub_name, child in module.named_children():
            fn_recursive_attn_processor(f'''{name}.{sub_name}'''						,				__lowercase						,				__lowercase )

        for name, module in self.named_children():
          fn_recursive_attn_processor(__lowercase						,				__lowercase						,				__lowercase )

      def       A_					(							self       :      Optional[Any] ):


        '''simple docstring'''




        self.set_attn_processor(AttnProcessor() )

      def       A_					(							self       :      Optional[int]						,				__lowercase       :      Dict						,				__lowercase       :      Union[torch.Tensor, float, int]						,				__lowercase       :      torch.FloatTensor						,				__lowercase       :      Optional[torch.FloatTensor] = None						,				__lowercase       :      Optional[torch.BoolTensor] = None						,				__lowercase       :      bool = True						,				):


        '''simple docstring'''




        __UpperCAmelCase    :						List[Any]       =						hidden_states.shape[0]

        __UpperCAmelCase    :						Any       =						timestep
        if not torch.is_tensor(__lowercase ):
          __UpperCAmelCase    :						Union[str, Any]       =						torch.tensor([timesteps]						,				dtype=torch.long						,				device=hidden_states.device )
        elif torch.is_tensor(__lowercase ) and len(timesteps.shape ) == 0:
          __UpperCAmelCase    :						Union[str, Any]       =						timesteps[None].to(hidden_states.device )

        # broadcast to batch dimension in a way that's compatible with ONNX/Core ML
        __UpperCAmelCase    :						List[str]       =						timesteps * torch.ones(__lowercase						,				dtype=timesteps.dtype						,				device=timesteps.device )

        __UpperCAmelCase    :						List[str]       =						self.time_proj(__lowercase )

        # timesteps does not contain any weights and will always return f32 tensors
        # but time_embedding might be fp16, so we need to cast here.
        __UpperCAmelCase    :						Any       =						timesteps_projected.to(dtype=self.dtype )
        __UpperCAmelCase    :						List[str]       =						self.time_embedding(__lowercase )

        if self.embedding_proj_norm is not None:
          __UpperCAmelCase    :						Dict       =						self.embedding_proj_norm(__lowercase )

        __UpperCAmelCase    :						Optional[int]       =						self.embedding_proj(__lowercase )
        if self.encoder_hidden_states_proj is not None and encoder_hidden_states is not None:
          __UpperCAmelCase    :						Dict       =						self.encoder_hidden_states_proj(__lowercase )
        elif self.encoder_hidden_states_proj is not None and encoder_hidden_states is None:
          raise ValueError('''`encoder_hidden_states_proj` requires `encoder_hidden_states` to be set''' )

        __UpperCAmelCase    :						int       =						self.proj_in(__lowercase )

        __UpperCAmelCase    :						Tuple       =						self.positional_embedding.to(hidden_states.dtype )

        __UpperCAmelCase    :						Optional[Any]       =						[]
        __UpperCAmelCase    :						str       =						0

        if encoder_hidden_states is not None:
          additional_embeds.append(__lowercase )
          additional_embeddings_len += encoder_hidden_states.shape[1]

        if len(proj_embeddings.shape ) == 2:
          __UpperCAmelCase    :						str       =						proj_embeddings[:, None, :]

        if len(hidden_states.shape ) == 2:
          __UpperCAmelCase    :						Optional[int]       =						hidden_states[:, None, :]

        __UpperCAmelCase    :						str       =						additional_embeds + [
            proj_embeddings,
            time_embeddings[:, None, :],
            hidden_states,
        ]

        if self.prd_embedding is not None:
          __UpperCAmelCase    :						Dict       =						self.prd_embedding.to(hidden_states.dtype ).expand(__lowercase						,				-1						,				-1 )
          additional_embeds.append(__lowercase )

        __UpperCAmelCase    :						Optional[Any]       =						torch.cat(
            __lowercase						,				dim=1						,				)

        # Allow positional_embedding to not include the `addtional_embeddings` and instead pad it with zeros for these additional tokens
        __UpperCAmelCase    :						Optional[int]       =						additional_embeddings_len + proj_embeddings.shape[1] + 1
        if positional_embeddings.shape[1] < hidden_states.shape[1]:
          __UpperCAmelCase    :						List[str]       =						F.pad(
              __lowercase						,				(
                  0,
                  0,
                  additional_embeddings_len,
                  self.prd_embedding.shape[1] if self.prd_embedding is not None else 0,
              )						,				value=0.0						,				)

        __UpperCAmelCase    :						Dict       =						hidden_states + positional_embeddings

        if attention_mask is not None:
          __UpperCAmelCase    :						Optional[Any]       =						(1 - attention_mask.to(hidden_states.dtype )) * -1_0_0_0_0.0
          __UpperCAmelCase    :						Optional[Any]       =						F.pad(__lowercase						,				(0, self.additional_embeddings)						,				value=0.0 )
          __UpperCAmelCase    :						Dict       =						(attention_mask[:, None, :] + self.causal_attention_mask).to(hidden_states.dtype )
          __UpperCAmelCase    :						Tuple       =						attention_mask.repeat_interleave(self.config.num_attention_heads						,				dim=0 )

        if self.norm_in is not None:
          __UpperCAmelCase    :						str       =						self.norm_in(__lowercase )

        for block in self.transformer_blocks:
          __UpperCAmelCase    :						Dict       =						block(__lowercase						,				attention_mask=__lowercase )

        __UpperCAmelCase    :						Any       =						self.norm_out(__lowercase )

        if self.prd_embedding is not None:
          __UpperCAmelCase    :						List[Any]       =						hidden_states[:, -1]
        else:
          __UpperCAmelCase    :						List[str]       =						hidden_states[:, additional_embeddings_len:]

        __UpperCAmelCase    :						Optional[int]       =						self.proj_to_clip_embeddings(__lowercase )

        if not return_dict:
          return (predicted_image_embedding,)

        return PriorTransformerOutput(predicted_image_embedding=__lowercase )







      def       A_					(							self       :      List[Any]						,				__lowercase       :      Optional[int] ):


        '''simple docstring'''




        __UpperCAmelCase    :						List[str]       =						(prior_latents * self.clip_std) + self.clip_mean
        return prior_latents