





"""simple docstring"""





import logging
import os
from dataclasses import dataclass
from enum import Enum
from typing import List, Optional, Union

from filelock import FileLock

from transformers import PreTrainedTokenizer, is_tf_available, is_torch_available


lowercase__				:List[Any]				   =       logging.getLogger(__name__)




@dataclass
class 							snake_case							:

      '''simple docstring'''



      _A     :       str
      _A     :       List[str]
      _A     :       Optional[List[str]]




@dataclass
class 							snake_case							:

      '''simple docstring'''



      _A     :       List[int]
      _A     :       List[int]
      _A     :       Optional[List[int]]         =       None
      _A     :       Optional[List[int]]         =       None




class 							snake_case							(			__UpperCAmelCase ):

      '''simple docstring'''



      _A     :       Optional[Any]         =       'train'
      _A     :       List[Any]         =       'dev'
      _A     :       Dict         =       'test'




class 							snake_case							:

      '''simple docstring'''



      @staticmethod
      def       A_					(							__lowercase       :      Any						,				__lowercase       :      Union[Split, str] ):


        '''simple docstring'''




        raise NotImplementedError

      @staticmethod
      def       A_					(							__lowercase       :      str ):


        '''simple docstring'''




        raise NotImplementedError







      @staticmethod
      def       A_					(							__lowercase       :      List[InputExample]						,				__lowercase       :      List[str]						,				__lowercase       :      int						,				__lowercase       :      PreTrainedTokenizer						,				__lowercase       :      Any=False						,				__lowercase       :      Tuple="[CLS]"						,				__lowercase       :      List[str]=1						,				__lowercase       :      List[Any]="[SEP]"						,				__lowercase       :      int=False						,				__lowercase       :      List[str]=False						,				__lowercase       :      Union[str, Any]=0						,				__lowercase       :      Optional[Any]=0						,				__lowercase       :      Tuple=-100						,				__lowercase       :      List[Any]=0						,				__lowercase       :      int=True						,				):


        '''simple docstring'''




        __UpperCAmelCase    :						int       =						{label: i for i, label in enumerate(__lowercase )}

        __UpperCAmelCase    :						str       =						[]
        for ex_index, example in enumerate(__lowercase ):
          if ex_index % 10_000 == 0:
            logger.info('''Writing example %d of %d'''						,				__lowercase						,				len(__lowercase ) )

          __UpperCAmelCase    :						str       =						[]
          __UpperCAmelCase    :						Any       =						[]
          for word, label in zip(example.words						,				example.labels ):
            __UpperCAmelCase    :						int       =						tokenizer.tokenize(__lowercase )

            # bert-base-multilingual-cased sometimes output "nothing ([]) when calling tokenize with just a space.
            if len(__lowercase ) > 0:
              tokens.extend(__lowercase )
              # Use the real label id for the first token of the word, and padding ids for the remaining tokens
              label_ids.extend([label_map[label]] + [pad_token_label_id] * (len(__lowercase ) - 1) )

            # Account for [CLS] and [SEP] with "- 2" and with "- 3" for RoBERTa.
          __UpperCAmelCase    :						Tuple       =						tokenizer.num_special_tokens_to_add()
          if len(__lowercase ) > max_seq_length - special_tokens_count:
            __UpperCAmelCase    :						List[Any]       =						tokens[: (max_seq_length - special_tokens_count)]
            __UpperCAmelCase    :						Tuple       =						label_ids[: (max_seq_length - special_tokens_count)]

          # The convention in BERT is:
          # (a) For sequence pairs:
          #  tokens:   [CLS] is this jack ##son ##ville ? [SEP] no it is not . [SEP]
          #  type_ids:   0   0  0    0    0     0       0   0   1  1  1  1   1   1
          # (b) For single sequences:
          #  tokens:   [CLS] the dog is hairy . [SEP]
          #  type_ids:   0   0   0   0  0     0   0
          #
          # Where "type_ids" are used to indicate whether this is the first
          # sequence or the second sequence. The embedding vectors for `type=0` and
          # `type=1` were learned during pre-training and are added to the wordpiece
          # embedding vector (and position vector). This is not *strictly* necessary
          # since the [SEP] token unambiguously separates the sequences, but it makes
          # it easier for the model to learn the concept of sequences.
          #
          # For classification tasks, the first vector (corresponding to [CLS]) is
          # used as the "sentence vector". Note that this only makes sense because
          # the entire model is fine-tuned.
          tokens += [sep_token]
          label_ids += [pad_token_label_id]
          if sep_token_extra:
            # roberta uses an extra separator b/w pairs of sentences
            tokens += [sep_token]
            label_ids += [pad_token_label_id]
          __UpperCAmelCase    :						Dict       =						[sequence_a_segment_id] * len(__lowercase )

          if cls_token_at_end:
            tokens += [cls_token]
            label_ids += [pad_token_label_id]
            segment_ids += [cls_token_segment_id]
          else:
            __UpperCAmelCase    :						Dict       =						[cls_token] + tokens
            __UpperCAmelCase    :						Tuple       =						[pad_token_label_id] + label_ids
            __UpperCAmelCase    :						Optional[Any]       =						[cls_token_segment_id] + segment_ids

          __UpperCAmelCase    :						Optional[Any]       =						tokenizer.convert_tokens_to_ids(__lowercase )

          # The mask has 1 for real tokens and 0 for padding tokens. Only real
          # tokens are attended to.
          __UpperCAmelCase    :						Optional[Any]       =						[1 if mask_padding_with_zero else 0] * len(__lowercase )

          # Zero-pad up to the sequence length.
          __UpperCAmelCase    :						Any       =						max_seq_length - len(__lowercase )
          if pad_on_left:
            __UpperCAmelCase    :						Optional[int]       =						([pad_token] * padding_length) + input_ids
            __UpperCAmelCase    :						List[str]       =						([0 if mask_padding_with_zero else 1] * padding_length) + input_mask
            __UpperCAmelCase    :						List[str]       =						([pad_token_segment_id] * padding_length) + segment_ids
            __UpperCAmelCase    :						Union[str, Any]       =						([pad_token_label_id] * padding_length) + label_ids
          else:
            input_ids += [pad_token] * padding_length
            input_mask += [0 if mask_padding_with_zero else 1] * padding_length
            segment_ids += [pad_token_segment_id] * padding_length
            label_ids += [pad_token_label_id] * padding_length

          assert len(__lowercase ) == max_seq_length
          assert len(__lowercase ) == max_seq_length
          assert len(__lowercase ) == max_seq_length
          assert len(__lowercase ) == max_seq_length

          if ex_index < 5:
            logger.info('''*** Example ***''' )
            logger.info('''guid: %s'''						,				example.guid )
            logger.info('''tokens: %s'''						,				''' '''.join([str(__lowercase ) for x in tokens] ) )
            logger.info('''input_ids: %s'''						,				''' '''.join([str(__lowercase ) for x in input_ids] ) )
            logger.info('''input_mask: %s'''						,				''' '''.join([str(__lowercase ) for x in input_mask] ) )
            logger.info('''segment_ids: %s'''						,				''' '''.join([str(__lowercase ) for x in segment_ids] ) )
            logger.info('''label_ids: %s'''						,				''' '''.join([str(__lowercase ) for x in label_ids] ) )

          if "token_type_ids" not in tokenizer.model_input_names:
            __UpperCAmelCase    :						int       =						None

          features.append(
              InputFeatures(
                  input_ids=__lowercase						,				attention_mask=__lowercase						,				token_type_ids=__lowercase						,				label_ids=__lowercase ) )
        return features


if is_torch_available():
  import torch
  from torch import nn
  from torch.utils.data import Dataset




  class 							snake_case							(			__UpperCAmelCase ):

        '''simple docstring'''



        _A     :       List[InputFeatures]
        _A     :       int         =       nn.CrossEntropyLoss().ignore_index

        def __init__(							self       :      Tuple						,				__lowercase       :      TokenClassificationTask						,				__lowercase       :      str						,				__lowercase       :      PreTrainedTokenizer						,				__lowercase       :      List[str]						,				__lowercase       :      str						,				__lowercase       :      Optional[int] = None						,				__lowercase       :      Optional[int]=False						,				__lowercase       :      Split = Split.train						,				):


          '''simple docstring'''




          __UpperCAmelCase    :						Optional[Any]       =						os.path.join(
              __lowercase						,				'''cached_{}_{}_{}'''.format(mode.value						,				tokenizer.__class__.__name__						,				str(__lowercase ) )						,				)

          # Make sure only the first process in distributed training processes the dataset,
          # and the others will use the cache.
          __UpperCAmelCase    :						Tuple       =						cached_features_file + '''.lock'''
          with FileLock(__lowercase ):
            if os.path.exists(__lowercase ) and not overwrite_cache:
              logger.info(f'''Loading features from cached file {cached_features_file}''' )
              __UpperCAmelCase    :						int       =						torch.load(__lowercase )
            else:
              logger.info(f'''Creating features from dataset file at {data_dir}''' )
              __UpperCAmelCase    :						Optional[Any]       =						token_classification_task.read_examples_from_file(__lowercase						,				__lowercase )
              # TODO clean up all this to leverage built-in features of tokenizers
              __UpperCAmelCase    :						Optional[int]       =						token_classification_task.convert_examples_to_features(
                  __lowercase						,				__lowercase						,				__lowercase						,				__lowercase						,				cls_token_at_end=bool(model_type in ['''xlnet'''] )						,				cls_token=tokenizer.cls_token						,				cls_token_segment_id=2 if model_type in ['''xlnet'''] else 0						,				sep_token=tokenizer.sep_token						,				sep_token_extra=__lowercase						,				pad_on_left=bool(tokenizer.padding_side == '''left''' )						,				pad_token=tokenizer.pad_token_id						,				pad_token_segment_id=tokenizer.pad_token_type_id						,				pad_token_label_id=self.pad_token_label_id						,				)
              logger.info(f'''Saving features into cached file {cached_features_file}''' )
              torch.save(self.features						,				__lowercase )

        def __len__(							self       :      List[str] ):


          '''simple docstring'''




          return len(self.features )







        def __getitem__(							self       :      Optional[Any]						,				__lowercase       :      int ):


          '''simple docstring'''




          return self.features[i]


if is_tf_available():
  import tensorflow as tf




  class 							snake_case							:

        '''simple docstring'''



        _A     :       List[InputFeatures]
        _A     :       int         =       -1_00

        def __init__(							self       :      Any						,				__lowercase       :      TokenClassificationTask						,				__lowercase       :      str						,				__lowercase       :      PreTrainedTokenizer						,				__lowercase       :      List[str]						,				__lowercase       :      str						,				__lowercase       :      Optional[int] = None						,				__lowercase       :      Dict=False						,				__lowercase       :      Split = Split.train						,				):


          '''simple docstring'''




          __UpperCAmelCase    :						List[Any]       =						token_classification_task.read_examples_from_file(__lowercase						,				__lowercase )
          # TODO clean up all this to leverage built-in features of tokenizers
          __UpperCAmelCase    :						Any       =						token_classification_task.convert_examples_to_features(
              __lowercase						,				__lowercase						,				__lowercase						,				__lowercase						,				cls_token_at_end=bool(model_type in ['''xlnet'''] )						,				cls_token=tokenizer.cls_token						,				cls_token_segment_id=2 if model_type in ['''xlnet'''] else 0						,				sep_token=tokenizer.sep_token						,				sep_token_extra=__lowercase						,				pad_on_left=bool(tokenizer.padding_side == '''left''' )						,				pad_token=tokenizer.pad_token_id						,				pad_token_segment_id=tokenizer.pad_token_type_id						,				pad_token_label_id=self.pad_token_label_id						,				)

          def gen():
            for ex in self.features:
              if ex.token_type_ids is None:
                yield (
                    {"input_ids": ex.input_ids, "attention_mask": ex.attention_mask},
                    ex.label_ids,
                )
              else:
                yield (
                    {
                        "input_ids": ex.input_ids,
                        "attention_mask": ex.attention_mask,
                        "token_type_ids": ex.token_type_ids,
                    },
                    ex.label_ids,
                )

          if "token_type_ids" not in tokenizer.model_input_names:
            __UpperCAmelCase    :						Optional[int]       =						tf.data.Dataset.from_generator(
                __lowercase						,				({'''input_ids''': tf.intaa, '''attention_mask''': tf.intaa}, tf.intaa)						,				(
                    {'''input_ids''': tf.TensorShape([None] ), '''attention_mask''': tf.TensorShape([None] )},
                    tf.TensorShape([None] ),
                )						,				)
          else:
            __UpperCAmelCase    :						Dict       =						tf.data.Dataset.from_generator(
                __lowercase						,				({'''input_ids''': tf.intaa, '''attention_mask''': tf.intaa, '''token_type_ids''': tf.intaa}, tf.intaa)						,				(
                    {
                        '''input_ids''': tf.TensorShape([None] ),
                        '''attention_mask''': tf.TensorShape([None] ),
                        '''token_type_ids''': tf.TensorShape([None] ),
                    },
                    tf.TensorShape([None] ),
                )						,				)

        def       A_					(							self       :      Optional[Any] ):


          '''simple docstring'''




          __UpperCAmelCase    :						Union[str, Any]       =						self.dataset.apply(tf.data.experimental.assert_cardinality(len(self.features ) ) )

          return self.dataset

        def __len__(							self       :      Tuple ):


          '''simple docstring'''




          return len(self.features )







        def __getitem__(							self       :      Optional[Any]						,				__lowercase       :      List[str] ):


          '''simple docstring'''




          return self.features[i]






"""simple docstring"""





from dataclasses import dataclass
from typing import Dict, Optional, Union

import torch
import torch.nn.functional as F
from torch import nn

from ..configuration_utils import ConfigMixin, register_to_config
from ..utils import BaseOutput
from .attention import BasicTransformerBlock
from .attention_processor import AttentionProcessor, AttnProcessor
from .embeddings import TimestepEmbedding, Timesteps
from .modeling_utils import ModelMixin




@dataclass
class 							snake_case							(			__UpperCAmelCase ):

      '''simple docstring'''



      _A     :       torch.FloatTensor




class 							snake_case							(			__UpperCAmelCase      ,  __UpperCAmelCase ):

      '''simple docstring'''



      @register_to_config
      def __init__(							self       :      Union[str, Any]						,				__lowercase       :      int = 32						,				__lowercase       :      int = 64						,				__lowercase       :      int = 20						,				__lowercase       :      int = 768						,				__lowercase       :      Optional[int]=77						,				__lowercase       :      Union[str, Any]=4						,				__lowercase       :      float = 0.0						,				__lowercase       :      str = "silu"						,				__lowercase       :      Optional[str] = None						,				__lowercase       :      Optional[str] = None						,				__lowercase       :      Optional[str] = "linear"						,				__lowercase       :      Optional[str] = "prd"						,				__lowercase       :      Optional[int] = None						,				__lowercase       :      Optional[int] = None						,				__lowercase       :      Optional[int] = None						,				):


        '''simple docstring'''




        super().__init__()
        __UpperCAmelCase    :						int       =						num_attention_heads
        __UpperCAmelCase    :						List[Any]       =						attention_head_dim
        __UpperCAmelCase    :						int       =						num_attention_heads * attention_head_dim
        __UpperCAmelCase    :						List[str]       =						additional_embeddings

        __UpperCAmelCase    :						Optional[int]       =						time_embed_dim or inner_dim
        __UpperCAmelCase    :						Tuple       =						embedding_proj_dim or embedding_dim
        __UpperCAmelCase    :						Dict       =						clip_embed_dim or embedding_dim

        __UpperCAmelCase    :						Dict       =						Timesteps(__lowercase						,				__lowercase						,				0 )
        __UpperCAmelCase    :						List[str]       =						TimestepEmbedding(__lowercase						,				__lowercase						,				out_dim=__lowercase						,				act_fn=__lowercase )

        __UpperCAmelCase    :						Any       =						nn.Linear(__lowercase						,				__lowercase )

        if embedding_proj_norm_type is None:
          __UpperCAmelCase    :						Dict       =						None
        elif embedding_proj_norm_type == "layer":
          __UpperCAmelCase    :						Any       =						nn.LayerNorm(__lowercase )
        else:
          raise ValueError(f'''unsupported embedding_proj_norm_type: {embedding_proj_norm_type}''' )

        __UpperCAmelCase    :						List[Any]       =						nn.Linear(__lowercase						,				__lowercase )

        if encoder_hid_proj_type is None:
          __UpperCAmelCase    :						int       =						None
        elif encoder_hid_proj_type == "linear":
          __UpperCAmelCase    :						int       =						nn.Linear(__lowercase						,				__lowercase )
        else:
          raise ValueError(f'''unsupported encoder_hid_proj_type: {encoder_hid_proj_type}''' )

        __UpperCAmelCase    :						str       =						nn.Parameter(torch.zeros(1						,				num_embeddings + additional_embeddings						,				__lowercase ) )

        if added_emb_type == "prd":
          __UpperCAmelCase    :						Optional[int]       =						nn.Parameter(torch.zeros(1						,				1						,				__lowercase ) )
        elif added_emb_type is None:
          __UpperCAmelCase    :						str       =						None
        else:
          raise ValueError(
              f'''`added_emb_type`: {added_emb_type} is not supported. Make sure to choose one of `\'prd\'` or `None`.''' )

        __UpperCAmelCase    :						Union[str, Any]       =						nn.ModuleList(
            [
                BasicTransformerBlock(
                    __lowercase						,				__lowercase						,				__lowercase						,				dropout=__lowercase						,				activation_fn='''gelu'''						,				attention_bias=__lowercase						,				)
                for d in range(__lowercase )
            ] )

        if norm_in_type == "layer":
          __UpperCAmelCase    :						Optional[Any]       =						nn.LayerNorm(__lowercase )
        elif norm_in_type is None:
          __UpperCAmelCase    :						List[Any]       =						None
        else:
          raise ValueError(f'''Unsupported norm_in_type: {norm_in_type}.''' )

        __UpperCAmelCase    :						str       =						nn.LayerNorm(__lowercase )

        __UpperCAmelCase    :						List[Any]       =						nn.Linear(__lowercase						,				__lowercase )

        __UpperCAmelCase    :						List[Any]       =						torch.full(
            [num_embeddings + additional_embeddings, num_embeddings + additional_embeddings]						,				-1_0_0_0_0.0 )
        causal_attention_mask.triu_(1 )
        __UpperCAmelCase    :						Optional[Any]       =						causal_attention_mask[None, ...]
        self.register_buffer('''causal_attention_mask'''						,				__lowercase						,				persistent=__lowercase )

        __UpperCAmelCase    :						Any       =						nn.Parameter(torch.zeros(1						,				__lowercase ) )
        __UpperCAmelCase    :						List[str]       =						nn.Parameter(torch.zeros(1						,				__lowercase ) )

      @property
      # Copied from diffusers.models.unet_2d_condition.UNet2DConditionModel.attn_processors
      def       A_					(							self       :      Optional[int] ):


        '''simple docstring'''




        __UpperCAmelCase    :						Optional[int]       =						{}

        def fn_recursive_add_processors(__lowercase       :      str						,				__lowercase       :      torch.nn.Module						,				__lowercase       :      Dict[str, AttentionProcessor] ):
          if hasattr(__lowercase						,				'''set_processor''' ):
            __UpperCAmelCase    :						Optional[Any]       =						module.processor

          for sub_name, child in module.named_children():
            fn_recursive_add_processors(f'''{name}.{sub_name}'''						,				__lowercase						,				__lowercase )

          return processors

        for name, module in self.named_children():
          fn_recursive_add_processors(__lowercase						,				__lowercase						,				__lowercase )

        return processors

      def       A_					(							self       :      Any						,				__lowercase       :      Union[AttentionProcessor, Dict[str, AttentionProcessor]] ):


        '''simple docstring'''




        __UpperCAmelCase    :						Tuple       =						len(self.attn_processors.keys() )

        if isinstance(__lowercase						,				__lowercase ) and len(__lowercase ) != count:
          raise ValueError(
              f'''A dict of processors was passed, but the number of processors {len(__lowercase )} does not match the'''
              f''' number of attention layers: {count}. Please make sure to pass {count} processor classes.''' )

        def fn_recursive_attn_processor(__lowercase       :      str						,				__lowercase       :      torch.nn.Module						,				__lowercase       :      int ):
          if hasattr(__lowercase						,				'''set_processor''' ):
            if not isinstance(__lowercase						,				__lowercase ):
              module.set_processor(__lowercase )
            else:
              module.set_processor(processor.pop(f'''{name}.processor''' ) )

          for sub_name, child in module.named_children():
            fn_recursive_attn_processor(f'''{name}.{sub_name}'''						,				__lowercase						,				__lowercase )

        for name, module in self.named_children():
          fn_recursive_attn_processor(__lowercase						,				__lowercase						,				__lowercase )

      def       A_					(							self       :      Optional[Any] ):


        '''simple docstring'''




        self.set_attn_processor(AttnProcessor() )

      def       A_					(							self       :      Optional[int]						,				__lowercase       :      Dict						,				__lowercase       :      Union[torch.Tensor, float, int]						,				__lowercase       :      torch.FloatTensor						,				__lowercase       :      Optional[torch.FloatTensor] = None						,				__lowercase       :      Optional[torch.BoolTensor] = None						,				__lowercase       :      bool = True						,				):


        '''simple docstring'''




        __UpperCAmelCase    :						List[Any]       =						hidden_states.shape[0]

        __UpperCAmelCase    :						Any       =						timestep
        if not torch.is_tensor(__lowercase ):
          __UpperCAmelCase    :						Union[str, Any]       =						torch.tensor([timesteps]						,				dtype=torch.long						,				device=hidden_states.device )
        elif torch.is_tensor(__lowercase ) and len(timesteps.shape ) == 0:
          __UpperCAmelCase    :						Union[str, Any]       =						timesteps[None].to(hidden_states.device )

        # broadcast to batch dimension in a way that's compatible with ONNX/Core ML
        __UpperCAmelCase    :						List[str]       =						timesteps * torch.ones(__lowercase						,				dtype=timesteps.dtype						,				device=timesteps.device )

        __UpperCAmelCase    :						List[str]       =						self.time_proj(__lowercase )

        # timesteps does not contain any weights and will always return f32 tensors
        # but time_embedding might be fp16, so we need to cast here.
        __UpperCAmelCase    :						Any       =						timesteps_projected.to(dtype=self.dtype )
        __UpperCAmelCase    :						List[str]       =						self.time_embedding(__lowercase )

        if self.embedding_proj_norm is not None:
          __UpperCAmelCase    :						Dict       =						self.embedding_proj_norm(__lowercase )

        __UpperCAmelCase    :						Optional[int]       =						self.embedding_proj(__lowercase )
        if self.encoder_hidden_states_proj is not None and encoder_hidden_states is not None:
          __UpperCAmelCase    :						Dict       =						self.encoder_hidden_states_proj(__lowercase )
        elif self.encoder_hidden_states_proj is not None and encoder_hidden_states is None:
          raise ValueError('''`encoder_hidden_states_proj` requires `encoder_hidden_states` to be set''' )

        __UpperCAmelCase    :						int       =						self.proj_in(__lowercase )

        __UpperCAmelCase    :						Tuple       =						self.positional_embedding.to(hidden_states.dtype )

        __UpperCAmelCase    :						Optional[Any]       =						[]
        __UpperCAmelCase    :						str       =						0

        if encoder_hidden_states is not None:
          additional_embeds.append(__lowercase )
          additional_embeddings_len += encoder_hidden_states.shape[1]

        if len(proj_embeddings.shape ) == 2:
          __UpperCAmelCase    :						str       =						proj_embeddings[:, None, :]

        if len(hidden_states.shape ) == 2:
          __UpperCAmelCase    :						Optional[int]       =						hidden_states[:, None, :]

        __UpperCAmelCase    :						str       =						additional_embeds + [
            proj_embeddings,
            time_embeddings[:, None, :],
            hidden_states,
        ]

        if self.prd_embedding is not None:
          __UpperCAmelCase    :						Dict       =						self.prd_embedding.to(hidden_states.dtype ).expand(__lowercase						,				-1						,				-1 )
          additional_embeds.append(__lowercase )

        __UpperCAmelCase    :						Optional[Any]       =						torch.cat(
            __lowercase						,				dim=1						,				)

        # Allow positional_embedding to not include the `addtional_embeddings` and instead pad it with zeros for these additional tokens
        __UpperCAmelCase    :						Optional[int]       =						additional_embeddings_len + proj_embeddings.shape[1] + 1
        if positional_embeddings.shape[1] < hidden_states.shape[1]:
          __UpperCAmelCase    :						List[str]       =						F.pad(
              __lowercase						,				(
                  0,
                  0,
                  additional_embeddings_len,
                  self.prd_embedding.shape[1] if self.prd_embedding is not None else 0,
              )						,				value=0.0						,				)

        __UpperCAmelCase    :						Dict       =						hidden_states + positional_embeddings

        if attention_mask is not None:
          __UpperCAmelCase    :						Optional[Any]       =						(1 - attention_mask.to(hidden_states.dtype )) * -1_0_0_0_0.0
          __UpperCAmelCase    :						Optional[Any]       =						F.pad(__lowercase						,				(0, self.additional_embeddings)						,				value=0.0 )
          __UpperCAmelCase    :						Dict       =						(attention_mask[:, None, :] + self.causal_attention_mask).to(hidden_states.dtype )
          __UpperCAmelCase    :						Tuple       =						attention_mask.repeat_interleave(self.config.num_attention_heads						,				dim=0 )

        if self.norm_in is not None:
          __UpperCAmelCase    :						str       =						self.norm_in(__lowercase )

        for block in self.transformer_blocks:
          __UpperCAmelCase    :						Dict       =						block(__lowercase						,				attention_mask=__lowercase )

        __UpperCAmelCase    :						Any       =						self.norm_out(__lowercase )

        if self.prd_embedding is not None:
          __UpperCAmelCase    :						List[Any]       =						hidden_states[:, -1]
        else:
          __UpperCAmelCase    :						List[str]       =						hidden_states[:, additional_embeddings_len:]

        __UpperCAmelCase    :						Optional[int]       =						self.proj_to_clip_embeddings(__lowercase )

        if not return_dict:
          return (predicted_image_embedding,)

        return PriorTransformerOutput(predicted_image_embedding=__lowercase )







      def       A_					(							self       :      List[Any]						,				__lowercase       :      Optional[int] ):


        '''simple docstring'''




        __UpperCAmelCase    :						List[str]       =						(prior_latents * self.clip_std) + self.clip_mean
        return prior_latents