


"""simple docstring"""





# DISCLAIMER: This code is strongly influenced by https://github.com/pesser/pytorch_diffusion
# and https://github.com/hojonathanho/diffusion
import math
from dataclasses import dataclass
from typing import List, Optional, Tuple, Union

import numpy as np
import torch

from diffusers.configuration_utils import ConfigMixin, register_to_config
from diffusers.schedulers.scheduling_utils import SchedulerMixin
from diffusers.utils import BaseOutput, deprecate

@dataclass
# Copied from diffusers.schedulers.scheduling_ddpm.DDPMSchedulerOutput with DDPM->DDIM
class 			lowercase_	(			UpperCAmelCase_    ):
      __magic_name__  :				torch.FloatTensor
      __magic_name__  :				Optional[torch.FloatTensor] 							=					None





def      lowercase__						(	lowerCamelCase  :	Any					,						lowerCamelCase  :	List[Any]=0.9_99					,						lowerCamelCase  :	str="cosine"					,						)       ->  Optional[Any]:
  if alpha_transform_type == "cosine":

    def alpha_bar_fn(lowerCamelCase  :	int					):
      return math.cos((t + 0.0_08) / 1.0_08 * math.pi / 2					) ** 2

  elif alpha_transform_type == "exp":

    def alpha_bar_fn(lowerCamelCase  :	List[Any]					):
      return math.exp(t * -12.0					)

  else:
    raise ValueError(F"Unsupported alpha_tranform_type: {alpha_transform_type}"					)

  lowerCAmelCase__     :     Tuple										=       []
  for i in range(__lowerCAmelCase					):
    lowerCAmelCase__     :     str										=       i / num_diffusion_timesteps
    lowerCAmelCase__     :     int										=       (i + 1) / num_diffusion_timesteps
    betas.append(min(1 - alpha_bar_fn(__lowerCAmelCase					) / alpha_bar_fn(__lowerCAmelCase					)					,						__lowerCAmelCase					)					)
  return torch.tensor(__lowerCAmelCase					,						dtype=torch.floataa					)







class 			lowercase_	(			UpperCAmelCase_				,				UpperCAmelCase_    ):
      __magic_name__  :				str 							=					1







      @register_to_config
      def __init__(   self			:				str		,	_lowercase			:				int = 1_0_0_0		,	_lowercase			:				float = 0.0001		,	_lowercase			:				float = 0.02		,	_lowercase			:				str = "linear"		,	_lowercase			:				Optional[Union[np.ndarray, List[float]]] = None		,	_lowercase			:				bool = True		,	_lowercase			:				bool = True		,	_lowercase			:				int = 0		,	_lowercase			:				str = "epsilon"		,	_lowercase			:				float = 1.0		,	**_lowercase			:				Optional[int]		,	):

        if kwargs.get("set_alpha_to_one"		,	lowerCamelCase__     ) is not None:
          lowerCAmelCase__     :     Optional[Any]										=       (
              """The `set_alpha_to_one` argument is deprecated. Please use `set_alpha_to_zero` instead."""
          )
          deprecate("set_alpha_to_one"		,	"1.0.0"		,	lowerCamelCase__		,	standard_warn=lowerCamelCase__     )
          lowerCAmelCase__     :     int										=       kwargs["""set_alpha_to_one"""]
        if trained_betas is not None:
          lowerCAmelCase__     :     Union[str, Any]										=       torch.tensor(lowerCamelCase__		,	dtype=torch.floataa     )
        elif beta_schedule == "linear":
          lowerCAmelCase__     :     Union[str, Any]										=       torch.linspace(lowerCamelCase__		,	lowerCamelCase__		,	lowerCamelCase__		,	dtype=torch.floataa     )
        elif beta_schedule == "scaled_linear":
          # this schedule is very specific to the latent diffusion model.
          lowerCAmelCase__     :     List[str]										=       (
              torch.linspace(beta_start**0.5		,	beta_end**0.5		,	lowerCamelCase__		,	dtype=torch.floataa     ) ** 2
          )
        elif beta_schedule == "squaredcos_cap_v2":
          # Glide cosine schedule
          lowerCAmelCase__     :     List[Any]										=       betas_for_alpha_bar(lowerCamelCase__     )
        else:
          raise NotImplementedError(f"{beta_schedule} does is not implemented for {self.__class__}"     )

        lowerCAmelCase__     :     List[Any]										=       1.0 - self.betas
        lowerCAmelCase__     :     Tuple										=       torch.cumprod(self.alphas		,	dim=0     )

        # At every step in inverted ddim, we are looking into the next alphas_cumprod
        # For the final step, there is no next alphas_cumprod, and the index is out of bounds
        # `set_alpha_to_zero` decides whether we set this parameter simply to zero
        # in this case, self.step() just output the predicted noise
        # or whether we use the final alpha of the "non-previous" one.
        lowerCAmelCase__     :     int										=       torch.tensor(0.0     ) if set_alpha_to_zero else self.alphas_cumprod[-1]

        # standard deviation of the initial noise distribution
        lowerCAmelCase__     :     Tuple										=       1.0

        # setable values
        lowerCAmelCase__     :     Optional[Any]										=       None
        lowerCAmelCase__     :     str										=       torch.from_numpy(np.arange(0		,	lowerCamelCase__     ).copy().astype(np.intaa     )     )







      def 					_lowerCAmelCase							(   self			:				Optional[int]		,	_lowercase			:				torch.FloatTensor		,	_lowercase			:				Optional[int] = None     ):
        return sample







      def 					_lowerCAmelCase							(   self			:				Tuple		,	_lowercase			:				int		,	_lowercase			:				Union[str, torch.device] = None     ):

        if num_inference_steps > self.config.num_train_timesteps:
          raise ValueError(
              f"`num_inference_steps`: {num_inference_steps} cannot be larger than `self.config.train_timesteps`:"
              f" {self.config.num_train_timesteps} as the unet model trained with this scheduler can only handle"
              f" maximal {self.config.num_train_timesteps} timesteps."     )

        lowerCAmelCase__     :     Any										=       num_inference_steps
        lowerCAmelCase__     :     Any										=       self.config.num_train_timesteps // self.num_inference_steps
        # creates integer timesteps by multiplying by ratio
        # casting to int to avoid issues when num_inference_step is power of 3
        lowerCAmelCase__     :     Union[str, Any]										=       (np.arange(0		,	lowerCamelCase__     ) * step_ratio).round().copy().astype(np.intaa     )
        lowerCAmelCase__     :     Any										=       torch.from_numpy(lowerCamelCase__     ).to(lowerCamelCase__     )
        self.timesteps += self.config.steps_offset







      def 					_lowerCAmelCase							(   self			:				Tuple		,	_lowercase			:				torch.FloatTensor		,	_lowercase			:				int		,	_lowercase			:				torch.FloatTensor		,	_lowercase			:				float = 0.0		,	_lowercase			:				bool = False		,	_lowercase			:				Optional[torch.FloatTensor] = None		,	_lowercase			:				bool = True		,	):
        lowerCAmelCase__     :     List[str]										=       timestep + self.config.num_train_timesteps // self.num_inference_steps

        # 2. compute alphas, betas
        # change original implementation to exactly match noise levels for analogous forward process
        lowerCAmelCase__     :     Any										=       self.alphas_cumprod[timestep]
        lowerCAmelCase__     :     Union[str, Any]										=       (
            self.alphas_cumprod[prev_timestep]
            if prev_timestep < self.config.num_train_timesteps
            else self.final_alpha_cumprod
        )

        lowerCAmelCase__     :     Tuple										=       1 - alpha_prod_t

        # 3. compute predicted original sample from predicted noise also called
        # "predicted x_0" of formula (12) from https://arxiv.org/pdf/2010.02502.pdf
        if self.config.prediction_type == "epsilon":
          lowerCAmelCase__     :     int										=       (sample - beta_prod_t ** 0.5 * model_output) / alpha_prod_t ** 0.5
          lowerCAmelCase__     :     List[str]										=       model_output
        elif self.config.prediction_type == "sample":
          lowerCAmelCase__     :     List[Any]										=       model_output
          lowerCAmelCase__     :     Any										=       (sample - alpha_prod_t ** 0.5 * pred_original_sample) / beta_prod_t ** 0.5
        elif self.config.prediction_type == "v_prediction":
          lowerCAmelCase__     :     List[str]										=       (alpha_prod_t**0.5) * sample - (beta_prod_t**0.5) * model_output
          lowerCAmelCase__     :     Optional[Any]										=       (alpha_prod_t**0.5) * model_output + (beta_prod_t**0.5) * sample
        else:
          raise ValueError(
              f"prediction_type given as {self.config.prediction_type} must be one of `epsilon`, `sample`, or"
              " `v_prediction`"     )

        # 4. Clip or threshold "predicted x_0"
        if self.config.clip_sample:
          lowerCAmelCase__     :     List[str]										=       pred_original_sample.clamp(
              -self.config.clip_sample_range		,	self.config.clip_sample_range     )

        # 5. compute "direction pointing to x_t" of formula (12) from https://arxiv.org/pdf/2010.02502.pdf
        lowerCAmelCase__     :     Optional[int]										=       (1 - alpha_prod_t_prev) ** 0.5 * pred_epsilon

        # 6. compute x_t without "random noise" of formula (12) from https://arxiv.org/pdf/2010.02502.pdf
        lowerCAmelCase__     :     Tuple										=       alpha_prod_t_prev ** 0.5 * pred_original_sample + pred_sample_direction

        if not return_dict:
          return (prev_sample, pred_original_sample)
        return DDIMSchedulerOutput(prev_sample=lowerCamelCase__		,	pred_original_sample=lowerCamelCase__     )







      def __len__(   self			:				List[Any]     ):
        return self.config.num_train_timesteps








import unittest

from transformers import AutoTokenizer, FalconConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
       import torch

       from transformers import (
           FalconForCausalLM,
           FalconForQuestionAnswering,
           FalconForSequenceClassification,
           FalconForTokenClassification,
           FalconModel,
       )







class 		_A						:

  def __init__(    self						:	int   ,				lowerCamelCase__						:	List[str]   ,				lowerCamelCase__						:	Union[str, Any]=3   ,				lowerCamelCase__						:	Optional[int]=7   ,				lowerCamelCase__						:	Optional[int]=True   ,				lowerCamelCase__						:	str=True   ,				lowerCamelCase__						:	Optional[int]=False   ,				lowerCamelCase__						:	Any=True   ,				lowerCamelCase__						:	int=99   ,				lowerCamelCase__						:	Optional[Any]=32   ,				lowerCamelCase__						:	Dict=5   ,				lowerCamelCase__						:	str=4   ,				lowerCamelCase__						:	Dict=37   ,				lowerCamelCase__						:	Union[str, Any]="gelu"   ,				lowerCamelCase__						:	Tuple=0.1   ,				lowerCamelCase__						:	str=0.1   ,				lowerCamelCase__						:	str=5_12   ,				lowerCamelCase__						:	int=16   ,				lowerCamelCase__						:	Optional[int]=2   ,				lowerCamelCase__						:	Any=0.02   ,				lowerCamelCase__						:	Any=3   ,				lowerCamelCase__						:	Tuple=4   ,				lowerCamelCase__						:	Any=None   ,				):
     """simple docstring"""

     __UpperCamelCase	:			Optional[Any]					=					parent
     __UpperCamelCase	:			Optional[Any]					=					batch_size
     __UpperCamelCase	:			Dict					=					seq_length
     __UpperCamelCase	:			Optional[int]					=					is_training
     __UpperCamelCase	:			str					=					use_input_mask
     __UpperCamelCase	:			Any					=					use_token_type_ids
     __UpperCamelCase	:			str					=					use_labels
     __UpperCamelCase	:			Tuple					=					vocab_size
     __UpperCamelCase	:			Dict					=					hidden_size
     __UpperCamelCase	:			Optional[int]					=					num_hidden_layers
     __UpperCamelCase	:			Optional[int]					=					num_attention_heads
     __UpperCamelCase	:			Optional[int]					=					intermediate_size
     __UpperCamelCase	:			List[Any]					=					hidden_act
     __UpperCamelCase	:			Any					=					hidden_dropout_prob
     __UpperCamelCase	:			Any					=					attention_probs_dropout_prob
     __UpperCamelCase	:			Optional[int]					=					max_position_embeddings
     __UpperCamelCase	:			Union[str, Any]					=					type_vocab_size
     __UpperCamelCase	:			List[Any]					=					type_sequence_label_size
     __UpperCamelCase	:			Union[str, Any]					=					initializer_range
     __UpperCamelCase	:			str					=					num_labels
     __UpperCamelCase	:			Tuple					=					num_choices
     __UpperCamelCase	:			Tuple					=					scope

  def    a					(    self						:	List[str]			):
     """simple docstring"""

     __UpperCamelCase	:			Union[str, Any]					=					ids_tensor([self.batch_size, self.seq_length]   ,				self.vocab_size			)

     __UpperCamelCase	:			Tuple					=					None
     if self.use_input_mask:
        __UpperCamelCase	:			Optional[int]					=					random_attention_mask([self.batch_size, self.seq_length]			)

     __UpperCamelCase	:			Optional[Any]					=					None

     __UpperCamelCase	:			Optional[Any]					=					None
     __UpperCamelCase	:			List[Any]					=					None
     __UpperCamelCase	:			Dict					=					None
     if self.use_labels:
        __UpperCamelCase	:			List[Any]					=					ids_tensor([self.batch_size]   ,				self.type_sequence_label_size			)
        __UpperCamelCase	:			Tuple					=					ids_tensor([self.batch_size, self.seq_length]   ,				self.num_labels			)
        __UpperCamelCase	:			List[Any]					=					ids_tensor([self.batch_size]   ,				self.num_choices			)

     __UpperCamelCase	:			str					=					self.get_config()

     return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels

  def    a					(    self						:	Dict			):
     """simple docstring"""

     return FalconConfig(
         vocab_size=self.vocab_size   ,				hidden_size=self.hidden_size   ,				num_hidden_layers=self.num_hidden_layers   ,				num_attention_heads=self.num_attention_heads   ,				intermediate_size=self.intermediate_size   ,				hidden_act=self.hidden_act   ,				hidden_dropout_prob=self.hidden_dropout_prob   ,				attention_probs_dropout_prob=self.attention_probs_dropout_prob   ,				max_position_embeddings=self.max_position_embeddings   ,				type_vocab_size=self.type_vocab_size   ,				is_decoder=lowerCamelCase__   ,				initializer_range=self.initializer_range   ,				pad_token_id=1   ,				new_decoder_architecture=lowerCamelCase__   ,				)

  def    a					(    self						:	Dict   ,				lowerCamelCase__						:	List[Any]   ,				lowerCamelCase__						:	Union[str, Any]   ,				lowerCamelCase__						:	Optional[int]   ,				lowerCamelCase__						:	Any   ,				lowerCamelCase__						:	Dict   ,				lowerCamelCase__						:	Dict   ,				lowerCamelCase__						:	int			):
     """simple docstring"""

     __UpperCamelCase	:			str					=					FalconModel(config=lowerCamelCase__			)
     model.to(lowerCamelCase__			)
     model.eval()
     __UpperCamelCase	:			List[Any]					=					model(lowerCamelCase__   ,				attention_mask=lowerCamelCase__			)
     __UpperCamelCase	:			Tuple					=					model(lowerCamelCase__			)
     self.parent.assertEqual(result.last_hidden_state.shape   ,				(self.batch_size, self.seq_length, self.hidden_size)			)

  def    a					(    self						:	Any   ,				lowerCamelCase__						:	List[Any]   ,				lowerCamelCase__						:	Any   ,				lowerCamelCase__						:	Union[str, Any]   ,				lowerCamelCase__						:	Union[str, Any]   ,				lowerCamelCase__						:	Optional[Any]   ,				lowerCamelCase__						:	Union[str, Any]   ,				lowerCamelCase__						:	Dict   ,				lowerCamelCase__						:	List[Any]   ,				lowerCamelCase__						:	int   ,				):
     """simple docstring"""

     __UpperCamelCase	:			str					=					True
     __UpperCamelCase	:			List[str]					=					FalconModel(lowerCamelCase__			)
     model.to(lowerCamelCase__			)
     model.eval()
     __UpperCamelCase	:			Dict					=					model(
         lowerCamelCase__   ,				attention_mask=lowerCamelCase__   ,				encoder_hidden_states=lowerCamelCase__   ,				encoder_attention_mask=lowerCamelCase__   ,				)
     __UpperCamelCase	:			Any					=					model(
         lowerCamelCase__   ,				attention_mask=lowerCamelCase__   ,				encoder_hidden_states=lowerCamelCase__   ,				)
     __UpperCamelCase	:			Any					=					model(lowerCamelCase__   ,				attention_mask=lowerCamelCase__			)
     self.parent.assertEqual(result.last_hidden_state.shape   ,				(self.batch_size, self.seq_length, self.hidden_size)			)

  def    a					(    self						:	Dict   ,				lowerCamelCase__						:	List[str]   ,				lowerCamelCase__						:	Tuple   ,				lowerCamelCase__						:	Dict   ,				lowerCamelCase__						:	Dict   ,				lowerCamelCase__						:	str   ,				lowerCamelCase__						:	str   ,				lowerCamelCase__						:	Tuple   ,				lowerCamelCase__						:	Any   ,				lowerCamelCase__						:	int   ,				):
     """simple docstring"""

     __UpperCamelCase	:			int					=					FalconForCausalLM(config=lowerCamelCase__			)
     model.to(lowerCamelCase__			)
     model.eval()
     __UpperCamelCase	:			Optional[Any]					=					model(lowerCamelCase__   ,				attention_mask=lowerCamelCase__   ,				labels=lowerCamelCase__			)
     self.parent.assertEqual(result.logits.shape   ,				(self.batch_size, self.seq_length, self.vocab_size)			)

  def    a					(    self						:	Optional[Any]   ,				lowerCamelCase__						:	Optional[int]   ,				lowerCamelCase__						:	Tuple   ,				lowerCamelCase__						:	Any   ,				lowerCamelCase__						:	str   ,				lowerCamelCase__						:	Union[str, Any]   ,				lowerCamelCase__						:	Dict   ,				lowerCamelCase__						:	Any   ,				lowerCamelCase__						:	int   ,				lowerCamelCase__						:	List[str]   ,				):
     """simple docstring"""

     __UpperCamelCase	:			List[str]					=					True
     __UpperCamelCase	:			List[Any]					=					True
     __UpperCamelCase	:			Dict					=					FalconForCausalLM(config=lowerCamelCase__			)
     model.to(lowerCamelCase__			)
     model.eval()

     # first forward pass
     __UpperCamelCase	:			Union[str, Any]					=					model(
         lowerCamelCase__   ,				attention_mask=lowerCamelCase__   ,				encoder_hidden_states=lowerCamelCase__   ,				encoder_attention_mask=lowerCamelCase__   ,				use_cache=lowerCamelCase__   ,				)
     __UpperCamelCase	:			List[Any]					=					outputs.past_key_values

     # create hypothetical multiple next token and extent to next_input_ids
     __UpperCamelCase	:			Optional[Any]					=					ids_tensor((self.batch_size, 3)   ,				config.vocab_size			)
     __UpperCamelCase	:			Dict					=					ids_tensor((self.batch_size, 3)   ,				vocab_size=2			)

     # append to next input_ids and
     __UpperCamelCase	:			Tuple					=					torch.cat([input_ids, next_tokens]   ,				dim=-1			)
     __UpperCamelCase	:			Union[str, Any]					=					torch.cat([input_mask, next_mask]   ,				dim=-1			)

     __UpperCamelCase	:			Optional[int]					=					model(
         lowerCamelCase__   ,				attention_mask=lowerCamelCase__   ,				encoder_hidden_states=lowerCamelCase__   ,				encoder_attention_mask=lowerCamelCase__   ,				output_hidden_states=lowerCamelCase__   ,				)["""hidden_states"""][0]
     __UpperCamelCase	:			Union[str, Any]					=					model(
         lowerCamelCase__   ,				attention_mask=lowerCamelCase__   ,				encoder_hidden_states=lowerCamelCase__   ,				encoder_attention_mask=lowerCamelCase__   ,				past_key_values=lowerCamelCase__   ,				output_hidden_states=lowerCamelCase__   ,				)["""hidden_states"""][0]

     # select random slice
     __UpperCamelCase	:			Dict					=					ids_tensor((1,)   ,				output_from_past.shape[-1]			).item()
     __UpperCamelCase	:			Dict					=					output_from_no_past[:, -3:, random_slice_idx].detach()
     __UpperCamelCase	:			Union[str, Any]					=					output_from_past[:, :, random_slice_idx].detach()

     self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1]			)

     # test that outputs are equal for slice
     self.parent.assertTrue(torch.allclose(lowerCamelCase__   ,				lowerCamelCase__   ,				atol=1e-3			)			)

  def    a					(    self						:	Union[str, Any]			):
     """simple docstring"""

     __UpperCamelCase	:			List[Any]					=					self.prepare_config_and_inputs()
     (
         (
         __UpperCamelCase
     )	,     (
         __UpperCamelCase
     )	,     (
         __UpperCamelCase
     )	,     (
         __UpperCamelCase
     )	,     (
         __UpperCamelCase
     )	,     (
         __UpperCamelCase
     )	,     (
         __UpperCamelCase
     )	,     
     )	:			Optional[Any]					=					config_and_inputs
     __UpperCamelCase	:			Any					=					{"""input_ids""": input_ids, """attention_mask""": input_mask}
     return config, inputs_dict







@require_torch
class 		_A						( UpperCAmelCase_	,				UpperCAmelCase_	,				UpperCAmelCase_	,				unittest.TestCase ):
  lowercase_						:			Any						=					(
      (
          FalconModel,
          FalconForCausalLM,
          FalconForSequenceClassification,
          FalconForTokenClassification,
          FalconForQuestionAnswering,
      )
      if is_torch_available()
      else ()
  )
  lowercase_						:			List[str]						=					(FalconForCausalLM,) if is_torch_available() else ()
  lowercase_						:			Union[str, Any]						=					(
      {
          '''feature-extraction''': FalconModel,
          '''text-classification''': FalconForSequenceClassification,
          '''text-generation''': FalconForCausalLM,
          '''question-answering''': FalconForQuestionAnswering,
          '''token-classification''': FalconForTokenClassification,
          '''zero-shot''': FalconForSequenceClassification,
      }
      if is_torch_available()
      else {}
  )
  lowercase_						:			Optional[int]						=					False
  lowercase_						:			Union[str, Any]						=					False

  def    a					(    self						:	List[str]			):
     """simple docstring"""

     __UpperCamelCase	:			Dict					=					FalconModelTester(self			)
     __UpperCamelCase	:			Dict					=					ConfigTester(self   ,				config_class=lowerCamelCase__   ,				hidden_size=37			)

  def    a					(    self						:	List[str]			):
     """simple docstring"""

     self.config_tester.run_common_tests()

  def    a					(    self						:	Union[str, Any]			):
     """simple docstring"""

     __UpperCamelCase	:			List[Any]					=					self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_model(*lowerCamelCase__			)

  def    a					(    self						:	Optional[Any]			):
     """simple docstring"""

     __UpperCamelCase	,     *__UpperCamelCase	:			List[Any]					=					self.model_tester.prepare_config_and_inputs()
     for alibi in [True, False]:
        __UpperCamelCase	:			List[str]					=					alibi
        self.model_tester.create_and_check_model(lowerCamelCase__   ,				*lowerCamelCase__			)

  def    a					(    self						:	int			):
     """simple docstring"""

     __UpperCamelCase	,     __UpperCamelCase	:			Optional[int]					=					self.model_tester.prepare_config_and_inputs_for_common()
     __UpperCamelCase	:			str					=					3
     __UpperCamelCase	:			Optional[int]					=					input_dict["""input_ids"""]
     __UpperCamelCase	:			List[Any]					=					input_ids.ne(1			).to(lowerCamelCase__			)
     __UpperCamelCase	:			List[str]					=					ids_tensor([self.model_tester.batch_size]   ,				self.model_tester.type_sequence_label_size			)
     __UpperCamelCase	:			Union[str, Any]					=					FalconForSequenceClassification(lowerCamelCase__			)
     model.to(lowerCamelCase__			)
     model.eval()
     __UpperCamelCase	:			int					=					model(lowerCamelCase__   ,				attention_mask=lowerCamelCase__   ,				labels=lowerCamelCase__			)
     self.assertEqual(result.logits.shape   ,				(self.model_tester.batch_size, self.model_tester.num_labels)			)

  def    a					(    self						:	Optional[Any]			):
     """simple docstring"""

     __UpperCamelCase	,     __UpperCamelCase	:			Any					=					self.model_tester.prepare_config_and_inputs_for_common()
     __UpperCamelCase	:			List[Any]					=					3
     __UpperCamelCase	:			List[Any]					=					"""single_label_classification"""
     __UpperCamelCase	:			Optional[Any]					=					input_dict["""input_ids"""]
     __UpperCamelCase	:			List[str]					=					input_ids.ne(1			).to(lowerCamelCase__			)
     __UpperCamelCase	:			Dict					=					ids_tensor([self.model_tester.batch_size]   ,				self.model_tester.type_sequence_label_size			)
     __UpperCamelCase	:			List[str]					=					FalconForSequenceClassification(lowerCamelCase__			)
     model.to(lowerCamelCase__			)
     model.eval()
     __UpperCamelCase	:			List[str]					=					model(lowerCamelCase__   ,				attention_mask=lowerCamelCase__   ,				labels=lowerCamelCase__			)
     self.assertEqual(result.logits.shape   ,				(self.model_tester.batch_size, self.model_tester.num_labels)			)

  def    a					(    self						:	List[Any]			):
     """simple docstring"""

     __UpperCamelCase	,     __UpperCamelCase	:			List[Any]					=					self.model_tester.prepare_config_and_inputs_for_common()
     __UpperCamelCase	:			Any					=					input_dict["""input_ids"""]
     __UpperCamelCase	:			List[str]					=					FalconForCausalLM(lowerCamelCase__			)
     model.to(lowerCamelCase__			)
     model.eval()
     __UpperCamelCase	:			Optional[Any]					=					model(lowerCamelCase__   ,				use_cache=lowerCamelCase__			)
     __UpperCamelCase	:			Optional[int]					=					input_ids.shape[0]
     __UpperCamelCase	:			Dict					=					model._convert_to_rw_cache(result.past_key_values			)
     __UpperCamelCase	:			Optional[int]					=					model._convert_cache_to_standard_format(lowerCamelCase__   ,				lowerCamelCase__			)
     for layer in range(len(lowerCamelCase__			)			):
        for tensor_idx in range(2			):
           self.assertTrue(rw_cache[layer][tensor_idx].ndim == 3			)
           self.assertTrue(result.past_key_values[layer][tensor_idx].ndim == 4			)
           self.assertTrue(
               torch.all(result.past_key_values[layer][tensor_idx] == standard_cache[layer][tensor_idx]			)			)

  def    a					(    self						:	Tuple			):
     """simple docstring"""

     __UpperCamelCase	,     __UpperCamelCase	:			Optional[int]					=					self.model_tester.prepare_config_and_inputs_for_common()
     __UpperCamelCase	:			Optional[Any]					=					3
     __UpperCamelCase	:			Optional[Any]					=					"""multi_label_classification"""
     __UpperCamelCase	:			Dict					=					input_dict["""input_ids"""]
     __UpperCamelCase	:			List[Any]					=					input_ids.ne(1			).to(lowerCamelCase__			)
     __UpperCamelCase	:			List[str]					=					ids_tensor(
         [self.model_tester.batch_size, config.num_labels]   ,				self.model_tester.type_sequence_label_size			).to(torch.float			)
     __UpperCamelCase	:			Tuple					=					FalconForSequenceClassification(lowerCamelCase__			)
     model.to(lowerCamelCase__			)
     model.eval()
     __UpperCamelCase	:			List[Any]					=					model(lowerCamelCase__   ,				attention_mask=lowerCamelCase__   ,				labels=lowerCamelCase__			)
     self.assertEqual(result.logits.shape   ,				(self.model_tester.batch_size, self.model_tester.num_labels)			)

  def    a					(    self						:	int			):
     """simple docstring"""

     for model_class in self.all_generative_model_classes:
        __UpperCamelCase	,     __UpperCamelCase	:			Union[str, Any]					=					self.model_tester.prepare_config_and_inputs_for_common()

        # If it doesn't support cache, pass the test
        if not hasattr(lowerCamelCase__   ,				"""use_cache"""			):
           return

        __UpperCamelCase	:			int					=					model_class(lowerCamelCase__			).to(lowerCamelCase__			)
        if "use_cache" not in inputs:
           __UpperCamelCase	:			Optional[Any]					=					True
        __UpperCamelCase	:			Any					=					model(**lowerCamelCase__			)

        # If "past_key_values" is not returned, pass the test (e.g. RWKV uses a different cache name and format)
        if "past_key_values" not in outputs:
           return

        __UpperCamelCase	:			List[str]					=					(
            getattr(lowerCamelCase__   ,				"""decoder_layers"""   ,				lowerCamelCase__			)
            or getattr(lowerCamelCase__   ,				"""num_decoder_layers"""   ,				lowerCamelCase__			)
            or config.num_hidden_layers
        )
        __UpperCamelCase	:			Optional[Any]					=					getattr(lowerCamelCase__   ,				"""num_kv_heads"""   ,				config.num_attention_heads			)
        __UpperCamelCase	:			str					=					getattr(lowerCamelCase__   ,				"""d_model"""   ,				config.hidden_size			)
        __UpperCamelCase	:			List[str]					=					embed_dim // num_attention_heads

        __UpperCamelCase	:			List[Any]					=					outputs["""past_key_values"""]
        self.assertEqual(len(lowerCamelCase__			)   ,				lowerCamelCase__			)

        __UpperCamelCase	,     __UpperCamelCase	:			Dict					=					inputs["""input_ids"""].shape
        for i in range(lowerCamelCase__			):
           if config.new_decoder_architecture:
              __UpperCamelCase	:			List[Any]					=					config.num_attention_heads
           elif config.multi_query:
              __UpperCamelCase	:			List[Any]					=					1
           self.assertEqual(len(past_kv[0]			)   ,				2			)  # K V for the decoder = 2
           self.assertEqual(
               past_kv[i][0].shape   ,				(batch_size, num_attention_heads, seq_length, per_head_embed_dim)			)
           self.assertEqual(
               past_kv[i][1].shape   ,				(batch_size, num_attention_heads, seq_length, per_head_embed_dim)			)







@require_torch
class 		_A						( unittest.TestCase ):

  @slow
  def    a					(    self						:	Optional[int]			):
     """simple docstring"""

     __UpperCamelCase	:			int					=					AutoTokenizer.from_pretrained("""Rocketknight1/falcon-rw-1b"""			)
     __UpperCamelCase	:			Optional[int]					=					FalconForCausalLM.from_pretrained("""Rocketknight1/falcon-rw-1b"""			)
     model.eval()
     model.to(lowerCamelCase__			)
     __UpperCamelCase	:			str					=					tokenizer("""My favorite food is"""   ,				return_tensors="""pt"""			).to(lowerCamelCase__			)

     __UpperCamelCase	:			Tuple					=					(
         """My favorite food is pizza. I love it so much that I have a pizza party every year for my birthday."""
     )

     __UpperCamelCase	:			Tuple					=					model.generate(**lowerCamelCase__   ,				do_sample=lowerCamelCase__   ,				max_new_tokens=19			)
     __UpperCamelCase	:			Dict					=					tokenizer.batch_decode(lowerCamelCase__			)[0]

     self.assertEqual(lowerCamelCase__   ,				lowerCamelCase__			)

  @slow
  def    a					(    self						:	Dict			):
     """simple docstring"""

     for repo in ["Rocketknight1/tiny-random-falcon-7b", "Rocketknight1/tiny-random-falcon-40b"]:
        __UpperCamelCase	:			Dict					=					AutoTokenizer.from_pretrained(lowerCamelCase__			)
        __UpperCamelCase	:			Union[str, Any]					=					FalconForCausalLM.from_pretrained(lowerCamelCase__			)
        model.eval()
        model.to(lowerCamelCase__			)
        __UpperCamelCase	:			Dict					=					tokenizer("""My favorite food is"""   ,				return_tensors="""pt"""			).to(lowerCamelCase__			)

        # We just test that these run without errors - the models are randomly initialized
        # and so the actual text outputs will be garbage
        model.generate(**lowerCamelCase__   ,				do_sample=lowerCamelCase__   ,				max_new_tokens=4			)
        model.generate(**lowerCamelCase__   ,				do_sample=lowerCamelCase__   ,				max_new_tokens=4			)
        model.generate(**lowerCamelCase__   ,				num_beams=2   ,				max_new_tokens=4			)

  @slow
  def    a					(    self						:	Tuple			):
     """simple docstring"""

     with torch.no_grad():
        for repo in [
            "Rocketknight1/falcon-rw-1b",
            "Rocketknight1/tiny-random-falcon-7b",
            "Rocketknight1/tiny-random-falcon-40b",
        ]:
           __UpperCamelCase	:			List[str]					=					AutoTokenizer.from_pretrained(lowerCamelCase__			)
           __UpperCamelCase	:			List[str]					=					FalconForCausalLM.from_pretrained(lowerCamelCase__			)
           model.eval()
           model.to(device=lowerCamelCase__			)
           __UpperCamelCase	:			Optional[int]					=					tokenizer("""My favorite food is"""   ,				return_tensors="""pt"""			).to(lowerCamelCase__			)

           # Test results are the same with and without cache
           __UpperCamelCase	:			Dict					=					model.generate(**lowerCamelCase__   ,				do_sample=lowerCamelCase__   ,				max_new_tokens=20   ,				use_cache=lowerCamelCase__			)
           __UpperCamelCase	:			int					=					model.generate(**lowerCamelCase__   ,				do_sample=lowerCamelCase__   ,				max_new_tokens=20   ,				use_cache=lowerCamelCase__			)
           self.assertTrue((outputs_cache - outputs_no_cache).sum().item() == 0			)



