





# DISCLAIMER: This code is strongly influenced by https://github.com/pesser/pytorch_diffusion
# and https://github.com/hojonathanho/diffusion
import math
from dataclasses import dataclass
from typing import List, Optional, Tuple, Union

import numpy as np
import torch

from diffusers.configuration_utils import ConfigMixin, register_to_config
from diffusers.schedulers.scheduling_utils import SchedulerMixin
from diffusers.utils import BaseOutput, deprecate

@dataclass
# Copied from diffusers.schedulers.scheduling_ddpm.DDPMSchedulerOutput with DDPM->DDIM
class       __A     (						a    ):
				__A      =				42
				__A      =				None





def      _lowercase					(						_UpperCAmelCase			,      _UpperCAmelCase=0.9_9_9			,      _UpperCAmelCase="cosine"			,      )						->  Optional[int]:
				if alpha_transform_type == "cosine":

								def alpha_bar_fn(_UpperCAmelCase				):
												return math.cos((t + 0.0_0_8) / 1.0_0_8 * math.pi / 2				) ** 2

				elif alpha_transform_type == "exp":

								def alpha_bar_fn(_UpperCAmelCase				):
												return math.exp(t * -1_2.0				)

				else:
								raise ValueError(F"""Unsupported alpha_tranform_type: {alpha_transform_type}"""				)

				lowerCamelCase	 =[]
				for i in range(_UpperCAmelCase				):
								lowerCamelCase	 =i / num_diffusion_timesteps
								lowerCamelCase	 =(i + 1) / num_diffusion_timesteps
								betas.append(min(1 - alpha_bar_fn(_UpperCAmelCase				) / alpha_bar_fn(_UpperCAmelCase				)			,      _UpperCAmelCase				)				)
				return torch.tensor(_UpperCAmelCase			,      dtype=torch.floataa				)




class       __A     (						a    ,					a    ):
				__A      =				1


				@register_to_config
				def __init__(		self   , UpperCAmelCase_ = 1000   , UpperCAmelCase_ = 0.0_0_0_1   , UpperCAmelCase_ = 0.0_2   , UpperCAmelCase_ = "linear"   , UpperCAmelCase_ = None   , UpperCAmelCase_ = True   , UpperCAmelCase_ = True   , UpperCAmelCase_ = 0   , UpperCAmelCase_ = "epsilon"   , UpperCAmelCase_ = 1.0   , **UpperCAmelCase_   , ):
								if kwargs.get("""set_alpha_to_one"""   , UpperCAmelCase_							) is not None:
												lowerCamelCase	 =(
												    """The `set_alpha_to_one` argument is deprecated. Please use `set_alpha_to_zero` instead."""
												)
												deprecate("""set_alpha_to_one"""   , """1.0.0"""   , UpperCAmelCase_   , standard_warn=UpperCAmelCase_							)
												lowerCamelCase	 =kwargs["""set_alpha_to_one"""]
								if trained_betas is not None:
												lowerCamelCase	 =torch.tensor(UpperCAmelCase_   , dtype=torch.floataa							)
								elif beta_schedule == "linear":
												lowerCamelCase	 =torch.linspace(UpperCAmelCase_   , UpperCAmelCase_   , UpperCAmelCase_   , dtype=torch.floataa							)
								elif beta_schedule == "scaled_linear":
												# this schedule is very specific to the latent diffusion model.
												lowerCamelCase	 =(
												    torch.linspace(beta_start**0.5   , beta_end**0.5   , UpperCAmelCase_   , dtype=torch.floataa							) ** 2
												)
								elif beta_schedule == "squaredcos_cap_v2":
												# Glide cosine schedule
												lowerCamelCase	 =betas_for_alpha_bar(UpperCAmelCase_							)
								else:
												raise NotImplementedError(f"""{beta_schedule} does is not implemented for {self.__class__}"""							)

								lowerCamelCase	 =1.0 - self.betas
								lowerCamelCase	 =torch.cumprod(self.alphas   , dim=0							)

								# At every step in inverted ddim, we are looking into the next alphas_cumprod
								# For the final step, there is no next alphas_cumprod, and the index is out of bounds
								# `set_alpha_to_zero` decides whether we set this parameter simply to zero
								# in this case, self.step() just output the predicted noise
								# or whether we use the final alpha of the "non-previous" one.
								lowerCamelCase	 =torch.tensor(0.0							) if set_alpha_to_zero else self.alphas_cumprod[-1]

								# standard deviation of the initial noise distribution
								lowerCamelCase	 =1.0

								# setable values
								lowerCamelCase	 =None
								lowerCamelCase	 =torch.from_numpy(np.arange(0   , UpperCAmelCase_							).copy().astype(np.intaa							)							)


				def _snake_case       (		self   , UpperCAmelCase_   , UpperCAmelCase_ = None							):
								return sample


				def _snake_case       (		self   , UpperCAmelCase_   , UpperCAmelCase_ = None							):

								if num_inference_steps > self.config.num_train_timesteps:
												raise ValueError(
												    f"""`num_inference_steps`: {num_inference_steps} cannot be larger than `self.config.train_timesteps`:"""
												    f""" {self.config.num_train_timesteps} as the unet model trained with this scheduler can only handle"""
												    f""" maximal {self.config.num_train_timesteps} timesteps."""							)

								lowerCamelCase	 =num_inference_steps
								lowerCamelCase	 =self.config.num_train_timesteps // self.num_inference_steps
								# creates integer timesteps by multiplying by ratio
								# casting to int to avoid issues when num_inference_step is power of 3
								lowerCamelCase	 =(np.arange(0   , UpperCAmelCase_							) * step_ratio).round().copy().astype(np.intaa							)
								lowerCamelCase	 =torch.from_numpy(UpperCAmelCase_							).to(UpperCAmelCase_							)
								self.timesteps += self.config.steps_offset


				def _snake_case       (		self   , UpperCAmelCase_   , UpperCAmelCase_   , UpperCAmelCase_   , UpperCAmelCase_ = 0.0   , UpperCAmelCase_ = False   , UpperCAmelCase_ = None   , UpperCAmelCase_ = True   , ):
								# 1. get previous step value (=t+1)
								lowerCamelCase	 =timestep + self.config.num_train_timesteps // self.num_inference_steps

								# 2. compute alphas, betas
								# change original implementation to exactly match noise levels for analogous forward process
								lowerCamelCase	 =self.alphas_cumprod[timestep]
								lowerCamelCase	 =(
								    self.alphas_cumprod[prev_timestep]
								    if prev_timestep < self.config.num_train_timesteps
								    else self.final_alpha_cumprod
								)

								lowerCamelCase	 =1 - alpha_prod_t

								# 3. compute predicted original sample from predicted noise also called
								# "predicted x_0" of formula (12) from https://arxiv.org/pdf/2010.02502.pdf
								if self.config.prediction_type == "epsilon":
												lowerCamelCase	 =(sample - beta_prod_t ** 0.5 * model_output) / alpha_prod_t ** 0.5
												lowerCamelCase	 =model_output
								elif self.config.prediction_type == "sample":
												lowerCamelCase	 =model_output
												lowerCamelCase	 =(sample - alpha_prod_t ** 0.5 * pred_original_sample) / beta_prod_t ** 0.5
								elif self.config.prediction_type == "v_prediction":
												lowerCamelCase	 =(alpha_prod_t**0.5) * sample - (beta_prod_t**0.5) * model_output
												lowerCamelCase	 =(alpha_prod_t**0.5) * model_output + (beta_prod_t**0.5) * sample
								else:
												raise ValueError(
												    f"""prediction_type given as {self.config.prediction_type} must be one of `epsilon`, `sample`, or"""
												    """ `v_prediction`"""							)

								# 4. Clip or threshold "predicted x_0"
								if self.config.clip_sample:
												lowerCamelCase	 =pred_original_sample.clamp(
												    -self.config.clip_sample_range   , self.config.clip_sample_range							)

								# 5. compute "direction pointing to x_t" of formula (12) from https://arxiv.org/pdf/2010.02502.pdf
								lowerCamelCase	 =(1 - alpha_prod_t_prev) ** 0.5 * pred_epsilon

								# 6. compute x_t without "random noise" of formula (12) from https://arxiv.org/pdf/2010.02502.pdf
								lowerCamelCase	 =alpha_prod_t_prev ** 0.5 * pred_original_sample + pred_sample_direction

								if not return_dict:
												return (prev_sample, pred_original_sample)
								return DDIMSchedulerOutput(prev_sample=UpperCAmelCase_   , pred_original_sample=UpperCAmelCase_							)


				def __len__(		self							):
								return self.config.num_train_timesteps












import json
import re
from typing import TYPE_CHECKING, List, Optional, Tuple, Union

import numpy as np

from ...utils import is_tf_available, is_torch_available, logging


if TYPE_CHECKING:
							if is_torch_available():
														import torch
							if is_tf_available():
														import tensorflow as tf

from tokenizers import pre_tokenizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from .tokenization_codegen import CodeGenTokenizer


UpperCAmelCase__						:			Optional[Any]									=logging.get_logger(__name__)

UpperCAmelCase__						:			Union[str, Any]									={'''vocab_file''': '''vocab.json''', '''merges_file''': '''merges.txt''', '''tokenizer_file''': '''tokenizer.json'''}

UpperCAmelCase__						:			str									={
    '''vocab_file''': {
        '''Salesforce/codegen-350M-mono''': '''https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/vocab.json''',
    },
    '''merges_file''': {
        '''Salesforce/codegen-350M-mono''': '''https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/merges.txt''',
    },
    '''tokenizer_file''': {
        '''Salesforce/codegen-350M-mono''': (
            '''https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/tokenizer.json'''
        ),
    },
}

UpperCAmelCase__						:			Union[str, Any]									={
    '''Salesforce/codegen-350M-mono''': 20_48,
}

class       __A     (						a    ):
				__A      =				VOCAB_FILES_NAMES
				__A      =				PRETRAINED_VOCAB_FILES_MAP
				__A      =				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				__A      =				["""input_ids""", """attention_mask"""]
				__A      =				CodeGenTokenizer


				def __init__(		self   , UpperCAmelCase_=None   , UpperCAmelCase_=None   , UpperCAmelCase_=None   , UpperCAmelCase_="<|endoftext|>"   , UpperCAmelCase_="<|endoftext|>"   , UpperCAmelCase_="<|endoftext|>"   , UpperCAmelCase_=False   , **UpperCAmelCase_   , ):
								super().__init__(
								    UpperCAmelCase_   , UpperCAmelCase_   , tokenizer_file=UpperCAmelCase_   , unk_token=UpperCAmelCase_   , bos_token=UpperCAmelCase_   , eos_token=UpperCAmelCase_   , add_prefix_space=UpperCAmelCase_   , **UpperCAmelCase_   , )

								if kwargs.pop("""add_bos_token"""   , UpperCAmelCase_							):
												lowerCamelCase	 =kwargs.pop("""name_or_path"""   , """"""							)
												raise ValueError(
												    """Currenty GPT2's fast tokenizer does NOT support adding a BOS token."""
												    """Instead you should use GPT2's slow tokenizer class `CodeGenTokenizer` as follows: \n"""
												    f"""`CodeGenTokenizer.from_pretrained('{model_id}')`\nor\n"""
												    f"""`AutoTokenizer.from_pretrained('{model_id}', use_fast=False)`\n"""
												    """This issue will be fixed soon, see: https://github.com/huggingface/tokenizers/pull/1005."""
												    """ so that the fast tokenizer works correctly."""							)

								lowerCamelCase	 =json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()							)
								if pre_tok_state.get("""add_prefix_space"""   , UpperCAmelCase_							) != add_prefix_space:
												lowerCamelCase	 =getattr(UpperCAmelCase_   , pre_tok_state.pop("""type"""							)							)
												lowerCamelCase	 =add_prefix_space
												lowerCamelCase	 =pre_tok_class(**UpperCAmelCase_							)

								lowerCamelCase	 =add_prefix_space


				def _snake_case       (		self   , *UpperCAmelCase_   , **UpperCAmelCase_							):
								lowerCamelCase	 =kwargs.get("""is_split_into_words"""   , UpperCAmelCase_							)
								assert self.add_prefix_space or not is_split_into_words, (
								    f"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
								    "to use it with pretokenized inputs."
								)

								return super()._batch_encode_plus(*UpperCAmelCase_   , **UpperCAmelCase_							)


				def _snake_case       (		self   , *UpperCAmelCase_   , **UpperCAmelCase_							):
								lowerCamelCase	 =kwargs.get("""is_split_into_words"""   , UpperCAmelCase_							)

								assert self.add_prefix_space or not is_split_into_words, (
								    f"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
								    "to use it with pretokenized inputs."
								)

								return super()._encode_plus(*UpperCAmelCase_   , **UpperCAmelCase_							)


				def _snake_case       (		self   , UpperCAmelCase_   , UpperCAmelCase_ = None							):
								lowerCamelCase	 =self._tokenizer.model.save(UpperCAmelCase_   , name=UpperCAmelCase_							)
								return tuple(UpperCAmelCase_							)


				def _snake_case       (		self   , UpperCAmelCase_   , UpperCAmelCase_ = False   , UpperCAmelCase_ = None   , UpperCAmelCase_ = None   , **UpperCAmelCase_   , ):
								lowerCamelCase	 =super().decode(
								    token_ids=UpperCAmelCase_   , skip_special_tokens=UpperCAmelCase_   , clean_up_tokenization_spaces=UpperCAmelCase_   , **UpperCAmelCase_   , )

								if truncate_before_pattern is not None and len(UpperCAmelCase_							) > 0:
												lowerCamelCase	 =self.truncate(UpperCAmelCase_   , UpperCAmelCase_							)

								return decoded_text


				def _snake_case       (		self   , UpperCAmelCase_   , UpperCAmelCase_							):
								def find_re(UpperCAmelCase_   , UpperCAmelCase_   , UpperCAmelCase_							):
												lowerCamelCase	 =pattern.search(UpperCAmelCase_   , UpperCAmelCase_							)
												return m.start() if m else -1

								lowerCamelCase	 =[re.compile(UpperCAmelCase_   , re.MULTILINE							) for pattern in truncate_before_pattern]

								lowerCamelCase	 =list(re.finditer("""^print"""   , UpperCAmelCase_   , re.MULTILINE							)							)

								if len(UpperCAmelCase_							) > 1:
												lowerCamelCase	 =completion[: prints[1].start()]

								lowerCamelCase	 =list(re.finditer("""^def"""   , UpperCAmelCase_   , re.MULTILINE							)							)

								if len(UpperCAmelCase_							) > 1:
												lowerCamelCase	 =completion[: defs[1].start()]

								lowerCamelCase	 =0

								lowerCamelCase	 =[
								    pos for pos in [find_re(UpperCAmelCase_   , UpperCAmelCase_   , UpperCAmelCase_							) for terminal in terminals] if pos != -1
								]

								if len(UpperCAmelCase_							) > 0:
												return completion[: min(UpperCAmelCase_							)]
								else:
												return completion





