





"""simple docstring"""
import torch
from torch import nn
from torch.nn import CrossEntropyLoss, MSELoss

from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward
from transformers.models.bert.modeling_bert import (
    BERT_INPUTS_DOCSTRING,
    BERT_START_DOCSTRING,
    BertEmbeddings,
    BertLayer,
    BertPooler,
    BertPreTrainedModel,
)

def 		_SCREAMING_SNAKE_CASE			(_UpperCAmelCase					:    List[str]			):
 lowerCAmelCase    				= torch.exp(_UpperCAmelCase			)
 lowerCAmelCase    				= torch.sum(_UpperCAmelCase  ,     dim=1			)  # sum of exp(x_i)
 lowerCAmelCase    				= torch.sum(x * exp_x  ,     dim=1			)  # sum of x_i * exp(x_i)
 return torch.log(_UpperCAmelCase			) - B / A







class       a							(   nn.Module  ):



 def __init__(     self				,       _snake_case     ):







  """simple docstring"""






  super().__init__()
  lowerCAmelCase    				= config.output_attentions
  lowerCAmelCase    				= config.output_hidden_states
  lowerCAmelCase    				= nn.ModuleList([BertLayer(_snake_case     ) for _ in range(config.num_hidden_layers     )]     )
  lowerCAmelCase    				= nn.ModuleList([BertHighway(_snake_case     ) for _ in range(config.num_hidden_layers     )]     )

  lowerCAmelCase    				= [-1 for _ in range(config.num_hidden_layers     )]



 def    UpperCamelCase__  (     self				,       _snake_case     ):







  """simple docstring"""






  if (type(_snake_case     ) is float) or (type(_snake_case     ) is int):
   for i in range(len(self.early_exit_entropy     )     ):
    lowerCAmelCase    				= x
  else:
   lowerCAmelCase    				= x



 def    UpperCamelCase__  (     self				,       _snake_case     ):







  """simple docstring"""






  lowerCAmelCase    				= pooler.state_dict()
  for highway in self.highway:
   for name, param in highway.pooler.state_dict().items():
    param.copy_(loaded_model[name]     )



 def    UpperCamelCase__  (     self				,       _snake_case				,       _snake_case=None				,       _snake_case=None				,       _snake_case=None				,       _snake_case=None				,       ):







  """simple docstring"""






  lowerCAmelCase    				= ()
  lowerCAmelCase    				= ()
  lowerCAmelCase    				= ()
  for i, layer_module in enumerate(self.layer     ):
   if self.output_hidden_states:
    lowerCAmelCase    				= all_hidden_states + (hidden_states,)

   lowerCAmelCase    				= layer_module(
       _snake_case				,       _snake_case				,       head_mask[i]				,       _snake_case				,       _snake_case     )
   lowerCAmelCase    				= layer_outputs[0]

   if self.output_attentions:
    lowerCAmelCase    				= all_attentions + (layer_outputs[1],)

   lowerCAmelCase    				= (hidden_states,)
   if self.output_hidden_states:
    lowerCAmelCase    				= current_outputs + (all_hidden_states,)
   if self.output_attentions:
    lowerCAmelCase    				= current_outputs + (all_attentions,)

   lowerCAmelCase    				= self.highway[i](_snake_case     )
   # logits, pooled_output

   if not self.training:
    lowerCAmelCase    				= highway_exit[0]
    lowerCAmelCase    				= entropy(_snake_case     )
    lowerCAmelCase    				= highway_exit + (highway_entropy,)  # logits, hidden_states(?), entropy
    lowerCAmelCase    				= all_highway_exits + (highway_exit,)

    if highway_entropy < self.early_exit_entropy[i]:
     lowerCAmelCase    				= (highway_logits,) + current_outputs[1:] + (all_highway_exits,)
     raise HighwayException(_snake_case				,       i + 1     )
   else:
    lowerCAmelCase    				= all_highway_exits + (highway_exit,)

        # Add last layer
  if self.output_hidden_states:
   lowerCAmelCase    				= all_hidden_states + (hidden_states,)

  lowerCAmelCase    				= (hidden_states,)
  if self.output_hidden_states:
   lowerCAmelCase    				= outputs + (all_hidden_states,)
  if self.output_attentions:
   lowerCAmelCase    				= outputs + (all_attentions,)

  lowerCAmelCase    				= outputs + (all_highway_exits,)
  return outputs  # last-layer hidden state, (all hidden states), (all attentions), all highway exits







@add_start_docstrings(
    '''The Bert Model transformer with early exiting (DeeBERT). '''   , a__   , )
class       a							(   a__  ):



 def __init__(     self				,       _snake_case     ):







  """simple docstring"""






  super().__init__(_snake_case     )
  lowerCAmelCase    				= config

  lowerCAmelCase    				= BertEmbeddings(_snake_case     )
  lowerCAmelCase    				= DeeBertEncoder(_snake_case     )
  lowerCAmelCase    				= BertPooler(_snake_case     )

  self.init_weights()



 def    UpperCamelCase__  (     self     ):







  """simple docstring"""






  self.encoder.init_highway_pooler(self.pooler     )



 def    UpperCamelCase__  (     self     ):







  """simple docstring"""






  return self.embeddings.word_embeddings



 def    UpperCamelCase__  (     self				,       _snake_case     ):







  """simple docstring"""






  lowerCAmelCase    				= value



 def    UpperCamelCase__  (     self				,       _snake_case     ):







  """simple docstring"""






  for layer, heads in heads_to_prune.items():
   self.encoder.layer[layer].attention.prune_heads(_snake_case     )



 @add_start_docstrings_to_model_forward(_snake_case     )
 def    UpperCamelCase__  (     self				,       _snake_case=None				,       _snake_case=None				,       _snake_case=None				,       _snake_case=None				,       _snake_case=None				,       _snake_case=None				,       _snake_case=None				,       _snake_case=None				,       ):







  """simple docstring"""






  if input_ids is not None and inputs_embeds is not None:
   raise ValueError('You cannot specify both input_ids and inputs_embeds at the same time'     )
  elif input_ids is not None:
   lowerCAmelCase    				= input_ids.size()
  elif inputs_embeds is not None:
   lowerCAmelCase    				= inputs_embeds.size()[:-1]
  else:
   raise ValueError('You have to specify either input_ids or inputs_embeds'     )

  lowerCAmelCase    				= input_ids.device if input_ids is not None else inputs_embeds.device

  if attention_mask is None:
   lowerCAmelCase    				= torch.ones(_snake_case				,       device=_snake_case     )
  if encoder_attention_mask is None:
   lowerCAmelCase    				= torch.ones(_snake_case				,       device=_snake_case     )
  if token_type_ids is None:
   lowerCAmelCase    				= torch.zeros(_snake_case				,       dtype=torch.long				,       device=_snake_case     )

  # We can provide a self-attention mask of dimensions [batch_size, from_seq_length, to_seq_length]
  # ourselves in which case we just need to make it broadcastable to all heads.
  lowerCAmelCase    				= self.get_extended_attention_mask(_snake_case				,       _snake_case				,       _snake_case     )

  # If a 2D ou 3D attention mask is provided for the cross-attention
  # we need to make broadcastable to [batch_size, num_heads, seq_length, seq_length]
  if encoder_attention_mask.dim() == 3:
   lowerCAmelCase    				= encoder_attention_mask[:, None, :, :]
  if encoder_attention_mask.dim() == 2:
   lowerCAmelCase    				= encoder_attention_mask[:, None, None, :]

  lowerCAmelCase    				= encoder_extended_attention_mask.to(
      dtype=next(self.parameters()     ).dtype     )  # fp16 compatibility
  lowerCAmelCase    				= (1.0 - encoder_extended_attention_mask) * -10_000.0

  # Prepare head mask if needed
  # 1.0 in head_mask indicate we keep the head
  # attention_probs has shape bsz x n_heads x N x N
  # input head_mask has shape [num_heads] or [num_hidden_layers x num_heads]
  # and head_mask is converted to shape [num_hidden_layers x batch x num_heads x seq_length x seq_length]
  lowerCAmelCase    				= self.get_head_mask(_snake_case				,       self.config.num_hidden_layers     )

  lowerCAmelCase    				= self.embeddings(
      input_ids=_snake_case				,       position_ids=_snake_case				,       token_type_ids=_snake_case				,       inputs_embeds=_snake_case     )
  lowerCAmelCase    				= self.encoder(
      _snake_case				,       attention_mask=_snake_case				,       head_mask=_snake_case				,       encoder_hidden_states=_snake_case				,       encoder_attention_mask=_snake_case				,       )
  lowerCAmelCase    				= encoder_outputs[0]
  lowerCAmelCase    				= self.pooler(_snake_case     )

  lowerCAmelCase    				= (
      sequence_output,
      pooled_output,
  ) + encoder_outputs[
      1:
  ]  # add hidden_states and attentions if they are here
  return outputs  # sequence_output, pooled_output, (hidden_states), (attentions), highway exits







class       a							(   a__  ):



 def __init__(     self				,       _snake_case				,       _snake_case     ):







  """simple docstring"""






  lowerCAmelCase    				= message
  lowerCAmelCase    				= exit_layer  # start from 1!







class       a							(   nn.Module  ):



 def __init__(     self				,       _snake_case     ):







  """simple docstring"""






  super().__init__()
  lowerCAmelCase    				= BertPooler(_snake_case     )
  lowerCAmelCase    				= nn.Dropout(config.hidden_dropout_prob     )
  lowerCAmelCase    				= nn.Linear(config.hidden_size				,       config.num_labels     )



 def    UpperCamelCase__  (     self				,       _snake_case     ):







  """simple docstring"""






  lowerCAmelCase    				= encoder_outputs[0]
  lowerCAmelCase    				= self.pooler(_snake_case     )
  # "return" pooler_output

  # BertModel
  lowerCAmelCase    				= (pooler_input, pooler_output) + encoder_outputs[1:]
  # "return" bmodel_output

  # Dropout and classification
  lowerCAmelCase    				= bmodel_output[1]

  lowerCAmelCase    				= self.dropout(_snake_case     )
  lowerCAmelCase    				= self.classifier(_snake_case     )

  return logits, pooled_output







@add_start_docstrings(
    '''Bert Model (with early exiting - DeeBERT) with a classifier on top,
    also takes care of multi-layer training. '''   , a__   , )
class       a							(   a__  ):



 def __init__(     self				,       _snake_case     ):







  """simple docstring"""






  super().__init__(_snake_case     )
  lowerCAmelCase    				= config.num_labels
  lowerCAmelCase    				= config.num_hidden_layers

  lowerCAmelCase    				= DeeBertModel(_snake_case     )
  lowerCAmelCase    				= nn.Dropout(config.hidden_dropout_prob     )
  lowerCAmelCase    				= nn.Linear(config.hidden_size				,       self.config.num_labels     )

  self.init_weights()



 @add_start_docstrings_to_model_forward(_snake_case     )
 def    UpperCamelCase__  (     self				,       _snake_case=None				,       _snake_case=None				,       _snake_case=None				,       _snake_case=None				,       _snake_case=None				,       _snake_case=None				,       _snake_case=None				,       _snake_case=-1				,       _snake_case=False				,       ):







  """simple docstring"""






  lowerCAmelCase    				= self.num_layers
  try:
   lowerCAmelCase    				= self.bert(
       _snake_case				,       attention_mask=_snake_case				,       token_type_ids=_snake_case				,       position_ids=_snake_case				,       head_mask=_snake_case				,       inputs_embeds=_snake_case				,       )
   # sequence_output, pooled_output, (hidden_states), (attentions), highway exits

   lowerCAmelCase    				= outputs[1]

   lowerCAmelCase    				= self.dropout(_snake_case     )
   lowerCAmelCase    				= self.classifier(_snake_case     )
   lowerCAmelCase    				= (logits,) + outputs[2:]  # add hidden states and attention if they are here
  except HighwayException as e:
   lowerCAmelCase    				= e.message
   lowerCAmelCase    				= e.exit_layer
   lowerCAmelCase    				= outputs[0]

  if not self.training:
   lowerCAmelCase    				= entropy(_snake_case     )
   lowerCAmelCase    				= []
   lowerCAmelCase    				= []
  if labels is not None:
   if self.num_labels == 1:
    #  We are doing regression
    lowerCAmelCase    				= MSELoss()
    lowerCAmelCase    				= loss_fct(logits.view(-1     )				,       labels.view(-1     )     )
   else:
    lowerCAmelCase    				= CrossEntropyLoss()
    lowerCAmelCase    				= loss_fct(logits.view(-1				,       self.num_labels     )				,       labels.view(-1     )     )

   # work with highway exits
   lowerCAmelCase    				= []
   for highway_exit in outputs[-1]:
    lowerCAmelCase    				= highway_exit[0]
    if not self.training:
     highway_logits_all.append(_snake_case     )
     highway_entropy.append(highway_exit[2]     )
    if self.num_labels == 1:
     #  We are doing regression
     lowerCAmelCase    				= MSELoss()
     lowerCAmelCase    				= loss_fct(highway_logits.view(-1     )				,       labels.view(-1     )     )
    else:
     lowerCAmelCase    				= CrossEntropyLoss()
     lowerCAmelCase    				= loss_fct(highway_logits.view(-1				,       self.num_labels     )				,       labels.view(-1     )     )
    highway_losses.append(_snake_case     )

   if train_highway:
    lowerCAmelCase    				= (sum(highway_losses[:-1]     ),) + outputs
    # exclude the final highway, of course
   else:
    lowerCAmelCase    				= (loss,) + outputs
  if not self.training:
   lowerCAmelCase    				= outputs + ((original_entropy, highway_entropy), exit_layer)
   if output_layer >= 0:
    lowerCAmelCase    				= (
        (outputs[0],) + (highway_logits_all[output_layer],) + outputs[2:]
    )  # use the highway of the last layer

  return outputs  # (loss), logits, (hidden_states), (attentions), (highway_exits)












"""simple docstring"""
import argparse
import math
import os

import torch
from neural_compressor.utils.pytorch import load
from PIL import Image
from transformers import CLIPTextModel, CLIPTokenizer

from diffusers import AutoencoderKL, StableDiffusionPipeline, UNetaDConditionModel

def 		_SCREAMING_SNAKE_CASE			():
 lowerCAmelCase    				= argparse.ArgumentParser()
 parser.add_argument(
     '-m'  ,     '--pretrained_model_name_or_path'  ,     type=_UpperCAmelCase  ,     default=_UpperCAmelCase  ,     required=_UpperCAmelCase  ,     help='Path to pretrained model or model identifier from huggingface.co/models.'  ,     )
 parser.add_argument(
     '-c'  ,     '--caption'  ,     type=_UpperCAmelCase  ,     default='robotic cat with wings'  ,     help='Text used to generate images.'  ,     )
 parser.add_argument(
     '-n'  ,     '--images_num'  ,     type=_UpperCAmelCase  ,     default=4  ,     help='How much images to generate.'  ,     )
 parser.add_argument(
     '-s'  ,     '--seed'  ,     type=_UpperCAmelCase  ,     default=42  ,     help='Seed for random process.'  ,     )
 parser.add_argument(
     '-ci'  ,     '--cuda_id'  ,     type=_UpperCAmelCase  ,     default=0  ,     help='cuda_id.'  ,     )
 lowerCAmelCase    				= parser.parse_args()
 return args

def 		_SCREAMING_SNAKE_CASE			(_UpperCAmelCase					:    int  ,     _UpperCAmelCase					:    List[Any]  ,     _UpperCAmelCase					:    Optional[int]			):
 if not len(_UpperCAmelCase			) == rows * cols:
  raise ValueError('The specified number of rows and columns are not correct.'			)

 lowerCAmelCase				,lowerCAmelCase    				= imgs[0].size
 lowerCAmelCase    				= Image.new('RGB'  ,     size=(cols * w, rows * h)			)
 lowerCAmelCase				,lowerCAmelCase    				= grid.size

 for i, img in enumerate(_UpperCAmelCase			):
  grid.paste(_UpperCAmelCase  ,     box=(i % cols * w, i // cols * h)			)
 return grid

def 		_SCREAMING_SNAKE_CASE			(_UpperCAmelCase					:    Any  ,     _UpperCAmelCase					:    Union[str, Any]="robotic cat with wings"  ,     _UpperCAmelCase					:    Optional[int]=7.5  ,     _UpperCAmelCase					:    Dict=50  ,     _UpperCAmelCase					:    Tuple=1  ,     _UpperCAmelCase					:    int=42  ,     ):
 lowerCAmelCase    				= torch.Generator(pipeline.device			).manual_seed(_UpperCAmelCase			)
 lowerCAmelCase    				= pipeline(
     _UpperCAmelCase  ,     guidance_scale=_UpperCAmelCase  ,     num_inference_steps=_UpperCAmelCase  ,     generator=_UpperCAmelCase  ,     num_images_per_prompt=_UpperCAmelCase  ,     ).images
 lowerCAmelCase    				= int(math.sqrt(_UpperCAmelCase			)			)
 lowerCAmelCase    				= image_grid(_UpperCAmelCase  ,     rows=_rows  ,     cols=num_images_per_prompt // _rows			)
 return grid, images


__UpperCamelCase     :						Optional[Any]						       =  parse_args()
# Load models and create wrapper for stable diffusion
__UpperCamelCase     :						List[Any]						       =  CLIPTokenizer.from_pretrained(args.pretrained_model_name_or_path, subfolder='''tokenizer''')
__UpperCamelCase     :						str						       =  CLIPTextModel.from_pretrained(args.pretrained_model_name_or_path, subfolder='''text_encoder''')
__UpperCamelCase     :						Optional[int]						       =  AutoencoderKL.from_pretrained(args.pretrained_model_name_or_path, subfolder='''vae''')
__UpperCamelCase     :						List[str]						       =  UNetaDConditionModel.from_pretrained(args.pretrained_model_name_or_path, subfolder='''unet''')

__UpperCamelCase     :						Tuple						       =  StableDiffusionPipeline.from_pretrained(
    args.pretrained_model_name_or_path, text_encoder=text_encoder, vae=vae, unet=unet, tokenizer=tokenizer
)
__UpperCamelCase     :						Union[str, Any]						       =  lambda images, clip_input: (images, False)
if os.path.exists(os.path.join(args.pretrained_model_name_or_path, '''best_model.pt''')):
       __UpperCamelCase     :						Dict						       =  load(args.pretrained_model_name_or_path, model=unet)
       unet.eval()
       setattr(pipeline, '''unet''', unet)
else:
       __UpperCamelCase     :						Dict						       =  unet.to(torch.device('''cuda''', args.cuda_id))
__UpperCamelCase     :						Optional[Any]						       =  pipeline.to(unet.device)
__UpperCamelCase       ,__UpperCamelCase     :						List[Any]						       =  generate_images(pipeline, prompt=args.caption, num_images_per_prompt=args.images_num, seed=args.seed)
grid.save(os.path.join(args.pretrained_model_name_or_path, '''{}.png'''.format('''_'''.join(args.caption.split()))))
__UpperCamelCase     :						int						       =  os.path.join(args.pretrained_model_name_or_path, '''_'''.join(args.caption.split()))
os.makedirs(dirname, exist_ok=True)
for idx, image in enumerate(images):
       image.save(os.path.join(dirname, '''{}.png'''.format(idx + 1)))





