'''simple docstring'''


from ...configuration_utils import PretrainedConfig
from ...utils import logging


snake_case__					:      List[str]				      =  logging.get_logger(__name__)

snake_case__					:      Optional[int]				      =  {
    '''tiiuae/falcon-40b''': '''https://huggingface.co/tiiuae/falcon-40b/resolve/main/config.json''',
    '''tiiuae/falcon-7b''': '''https://huggingface.co/tiiuae/falcon-7b/resolve/main/config.json''',
}






class 					__SCREAMING_SNAKE_CASE						(	lowerCamelCase_	):







  '''simple docstring'''






  lowerCamelCase_		:str						   =		'''falcon'''
  lowerCamelCase_		:Optional[Any]						   =		['''past_key_values''']


  def __init__(			self				,		snake_case_=6_5_0_2_4				,		snake_case_=4_5_4_4				,		snake_case_=3_2				,		snake_case_=7_1				,		snake_case_=1E-5				,		snake_case_=0.02				,		snake_case_=True				,		snake_case_=0.0				,		snake_case_=0.0				,		snake_case_=None				,		snake_case_=False				,		snake_case_=False				,		snake_case_=True				,		snake_case_=True				,		snake_case_=False				,		snake_case_=1_1				,		snake_case_=1_1				,		**snake_case_				,		):




       '''simple docstring'''

       UpperCAmelCase_							:       Optional[int]       =				vocab_size
       # Backward compatibility with n_embed kwarg
       UpperCAmelCase_							:       Optional[Any]       =				kwargs.pop('n_embed'				,		snake_case_						)
       UpperCAmelCase_							:       int       =				hidden_size if n_embed is None else n_embed
       UpperCAmelCase_							:       Optional[Any]       =				num_hidden_layers
       UpperCAmelCase_							:       Any       =				num_attention_heads
       UpperCAmelCase_							:       Optional[int]       =				layer_norm_epsilon
       UpperCAmelCase_							:       Optional[Any]       =				initializer_range
       UpperCAmelCase_							:       Dict       =				use_cache
       UpperCAmelCase_							:       List[Any]       =				hidden_dropout
       UpperCAmelCase_							:       List[Any]       =				attention_dropout

       UpperCAmelCase_							:       Union[str, Any]       =				bos_token_id
       UpperCAmelCase_							:       Optional[int]       =				eos_token_id
       UpperCAmelCase_							:       int       =				num_attention_heads if num_kv_heads is None else num_kv_heads
       UpperCAmelCase_							:       Dict       =				alibi
       UpperCAmelCase_							:       List[str]       =				new_decoder_architecture
       UpperCAmelCase_							:       List[str]       =				multi_query  # Ignored when new_decoder_architecture is True
       UpperCAmelCase_							:       List[str]       =				parallel_attn
       UpperCAmelCase_							:       Union[str, Any]       =				bias

       super().__init__(bos_token_id=snake_case_				,		eos_token_id=snake_case_				,		**snake_case_						)


  @property
  def _UpperCamelCase					(			self						):




       '''simple docstring'''

       return self.hidden_size // self.num_attention_heads






  @property
  def _UpperCamelCase					(			self						):




       '''simple docstring'''

       return not self.alibi







'''simple docstring'''


import math
from typing import List, Optional, Tuple, Union

import numpy as np
import torch

from ..configuration_utils import ConfigMixin, register_to_config
from .scheduling_utils import SchedulerMixin, SchedulerOutput






class 					__SCREAMING_SNAKE_CASE						(	lowerCamelCase_						,     lowerCamelCase_	):







  '''simple docstring'''






  lowerCamelCase_		:Union[str, Any]						   =		1


  @register_to_config
  def __init__(			self				,		snake_case_ = 1_0_0_0				,		snake_case_ = None						):




       '''simple docstring'''

       self.set_timesteps(snake_case_						)

       # standard deviation of the initial noise distribution
       UpperCAmelCase_							:       Union[str, Any]       =				1.0

       # For now we only support F-PNDM, i.e. the runge-kutta method
       # For more information on the algorithm please take a look at the paper: https://arxiv.org/pdf/2202.09778.pdf
       # mainly at formula (9), (12), (13) and the Algorithm 2.
       UpperCAmelCase_							:       int       =				4

       # running values
       UpperCAmelCase_							:       str       =				[]


  def _UpperCamelCase					(			self				,		snake_case_				,		snake_case_ = None						):




       '''simple docstring'''

       UpperCAmelCase_							:       Union[str, Any]       =				num_inference_steps
       UpperCAmelCase_							:       int       =				torch.linspace(1				,		0				,		num_inference_steps + 1						)[:-1]
       UpperCAmelCase_							:       Tuple       =				torch.cat([steps, torch.tensor([0.0]						)]						)

       if self.config.trained_betas is not None:
            UpperCAmelCase_							:       Optional[int]       =				torch.tensor(self.config.trained_betas				,		dtype=torch.floataa						)
       else:
            UpperCAmelCase_							:       Tuple       =				torch.sin(steps * math.pi / 2						) ** 2

       UpperCAmelCase_							:       Dict       =				(1.0 - self.betas**2) ** 0.5

       UpperCAmelCase_							:       str       =				(torch.atana(self.betas				,		self.alphas						) / math.pi * 2)[:-1]
       UpperCAmelCase_							:       str       =				timesteps.to(snake_case_						)

       UpperCAmelCase_							:       Any       =				[]


  def _UpperCamelCase					(			self				,		snake_case_				,		snake_case_				,		snake_case_				,		snake_case_ = True				,		):




       '''simple docstring'''

       if self.num_inference_steps is None:
            raise ValueError(
                'Number of inference steps is \'None\', you need to run \'set_timesteps\' after creating the scheduler'						)

       UpperCAmelCase_							:       Any       =				(self.timesteps == timestep).nonzero().item()
       UpperCAmelCase_							:       Optional[Any]       =				timestep_index + 1

       UpperCAmelCase_							:       Dict       =				sample * self.betas[timestep_index] + model_output * self.alphas[timestep_index]
       self.ets.append(snake_case_						)

       if len(self.ets						) == 1:
            UpperCAmelCase_							:       Tuple       =				self.ets[-1]
       elif len(self.ets						) == 2:
            UpperCAmelCase_							:       Any       =				(3 * self.ets[-1] - self.ets[-2]) / 2
       elif len(self.ets						) == 3:
            UpperCAmelCase_							:       List[str]       =				(2_3 * self.ets[-1] - 1_6 * self.ets[-2] + 5 * self.ets[-3]) / 1_2
       else:
            UpperCAmelCase_							:       Union[str, Any]       =				(1 / 2_4) * (5_5 * self.ets[-1] - 5_9 * self.ets[-2] + 3_7 * self.ets[-3] - 9 * self.ets[-4])

       UpperCAmelCase_							:       Union[str, Any]       =				self._get_prev_sample(snake_case_				,		snake_case_				,		snake_case_				,		snake_case_						)

       if not return_dict:
            return (prev_sample,)

       return SchedulerOutput(prev_sample=snake_case_						)


  def _UpperCamelCase					(			self				,		snake_case_				,		*snake_case_				,		**snake_case_						):




       '''simple docstring'''

       return sample


  def _UpperCamelCase					(			self				,		snake_case_				,		snake_case_				,		snake_case_				,		snake_case_						):




       '''simple docstring'''

       UpperCAmelCase_							:       int       =				self.alphas[timestep_index]
       UpperCAmelCase_							:       Union[str, Any]       =				self.betas[timestep_index]

       UpperCAmelCase_							:       Any       =				self.alphas[prev_timestep_index]
       UpperCAmelCase_							:       Dict       =				self.betas[prev_timestep_index]

       UpperCAmelCase_							:       List[Any]       =				(sample - sigma * ets) / max(snake_case_				,		1E-8						)
       UpperCAmelCase_							:       Tuple       =				next_alpha * pred + ets * next_sigma

       return prev_sample






  def __len__(			self						):




       '''simple docstring'''

       return self.config.num_train_timesteps






