



'''simple docstring'''





import inspect

import jax
import jax.lax as lax
import jax.numpy as jnp

from ..utils import add_start_docstrings
from ..utils.logging import get_logger


SCREAMING_SNAKE_CASE__      					=     get_logger(__name__)


SCREAMING_SNAKE_CASE__      					=     R"\n    Args:\n        input_ids (`jnp.ndarray` of shape `(batch_size, sequence_length)`):\n            Indices of input sequence tokens in the vocabulary.\n\n            Indices can be obtained using [`PreTrainedTokenizer`]. See [`PreTrainedTokenizer.encode`] and\n            [`PreTrainedTokenizer.__call__`] for details.\n\n            [What are input IDs?](../glossary#input-ids)\n        scores (`jnp.ndarray` of shape `(batch_size, config.vocab_size)`):\n            Prediction scores of a language modeling head. These can be logits for each vocabulary when not using beam\n            search or log softmax for each vocabulary token when using beam search\n        kwargs (`Dict[str, Any]`, *optional*):\n            Additional logits processor specific kwargs.\n\n    Return:\n        `jnp.ndarray` of shape `(batch_size, config.vocab_size)`: The processed prediction scores.\n\n"


class 				snake_case :



       @add_start_docstrings(UpperCAmelCase_      )
       def __call__(				self     ,UpperCAmelCase_     ,UpperCAmelCase_      )  ->							jnp.ndarray:
           raise NotImplementedError(
               F'''{self.__class__} is an abstract class. Only classes inheriting this class can be called.'''      )





class 				snake_case :



       @add_start_docstrings(UpperCAmelCase_      )
       def __call__(				self     ,UpperCAmelCase_     ,UpperCAmelCase_      )  ->							jnp.ndarray:
           raise NotImplementedError(
               F'''{self.__class__} is an abstract class. Only classes inheriting this class can be called.'''      )





class 				snake_case (UpperCamelCase   ):



       @add_start_docstrings(UpperCAmelCase_      )
       def __call__(				self     ,UpperCAmelCase_     ,UpperCAmelCase_     ,UpperCAmelCase_     ,**UpperCAmelCase_      )  ->							jnp.ndarray:
           for processor in self:
               lowercase__		       =						inspect.signature(processor.__call__      ).parameters
               if len(UpperCAmelCase_      ) > 3:
                   if not all(arg in kwargs for arg in list(function_args.keys()      )[2:]      ):
                       raise ValueError(
                           F'''Make sure that all the required parameters: {list(function_args.keys()      )} for '''
                           F'''{processor.__class__} are passed to the logits processor.'''      )
                   lowercase__		       =						processor(UpperCAmelCase_     ,UpperCAmelCase_     ,UpperCAmelCase_     ,**UpperCAmelCase_      )
               else:
                   lowercase__		       =						processor(UpperCAmelCase_     ,UpperCAmelCase_     ,UpperCAmelCase_      )
           return scores





class 				snake_case (UpperCamelCase   ):



       def __init__(				self     ,UpperCAmelCase_      )  ->							str:
           if not isinstance(UpperCAmelCase_     ,UpperCAmelCase_      ) or not (temperature > 0):
               raise ValueError(F'''`temperature` has to be a strictly positive float, but is {temperature}'''      )

           lowercase__		       =						temperature







       def __call__(				self     ,UpperCAmelCase_     ,UpperCAmelCase_     ,UpperCAmelCase_      )  ->							jnp.ndarray:
           lowercase__		       =						scores / self.temperature
           return scores





class 				snake_case (UpperCamelCase   ):



       def __init__(				self     ,UpperCAmelCase_     ,UpperCAmelCase_ = -float("Inf"      )     ,UpperCAmelCase_ = 1      )  ->							Optional[int]:
           if not isinstance(UpperCAmelCase_     ,UpperCAmelCase_      ) or (top_p < 0 or top_p > 1.0):
               raise ValueError(F'''`top_p` has to be a float > 0 and < 1, but is {top_p}'''      )
           if not isinstance(UpperCAmelCase_     ,UpperCAmelCase_      ) or (min_tokens_to_keep < 1):
               raise ValueError(F'''`min_tokens_to_keep` has to be a positive integer, but is {min_tokens_to_keep}'''      )

           lowercase__		       =						top_p
           lowercase__		       =						filter_value
           lowercase__		       =						min_tokens_to_keep







       def __call__(				self     ,UpperCAmelCase_     ,UpperCAmelCase_     ,UpperCAmelCase_      )  ->							jnp.ndarray:
           lowercase__       ,       lowercase__		       =						lax.top_k(UpperCAmelCase_     ,scores.shape[-1]      )

           lowercase__		       =						jnp.full_like(UpperCAmelCase_     ,self.filter_value      )
           lowercase__		       =						jax.nn.softmax(UpperCAmelCase_     ,axis=-1      ).cumsum(axis=-1      )
           lowercase__		       =						cumulative_probs < self.top_p

           # include the token that is higher than top_p as well
           lowercase__		       =						jnp.roll(UpperCAmelCase_     ,1      )
           score_mask |= score_mask.at[:, 0].set(UpperCAmelCase_      )

           # min tokens to keep
           lowercase__		       =						score_mask.at[:, : self.min_tokens_to_keep].set(UpperCAmelCase_      )

           lowercase__		       =						jnp.where(UpperCAmelCase_     ,UpperCAmelCase_     ,UpperCAmelCase_      )
           lowercase__		       =						jax.lax.sort_key_val(UpperCAmelCase_     ,UpperCAmelCase_      )[-1]

           return next_scores





class 				snake_case (UpperCamelCase   ):



       def __init__(				self     ,UpperCAmelCase_     ,UpperCAmelCase_ = -float("Inf"      )     ,UpperCAmelCase_ = 1      )  ->							Optional[Any]:
           if not isinstance(UpperCAmelCase_     ,UpperCAmelCase_      ) or top_k <= 0:
               raise ValueError(F'''`top_k` has to be a strictly positive integer, but is {top_k}'''      )

           lowercase__		       =						max(UpperCAmelCase_     ,UpperCAmelCase_      )
           lowercase__		       =						filter_value







       def __call__(				self     ,UpperCAmelCase_     ,UpperCAmelCase_     ,UpperCAmelCase_      )  ->							jnp.ndarray:
           lowercase__       ,       lowercase__		       =						scores.shape
           lowercase__		       =						jnp.full(batch_size * vocab_size     ,self.filter_value      )

           lowercase__		       =						min(self.top_k     ,scores.shape[-1]      )  # Safety check
           lowercase__       ,       lowercase__		       =						lax.top_k(UpperCAmelCase_     ,UpperCAmelCase_      )
           lowercase__		       =						jnp.broadcast_to((jnp.arange(UpperCAmelCase_      ) * vocab_size)[:, None]     ,(batch_size, topk)      ).flatten()
           lowercase__		       =						topk_scores.flatten()
           lowercase__		       =						topk_indices.flatten() + shift

           lowercase__		       =						next_scores_flat.at[topk_indices_flat].set(UpperCAmelCase_      )
           lowercase__		       =						next_scores_flat.reshape(UpperCAmelCase_     ,UpperCAmelCase_      )
           return next_scores





class 				snake_case (UpperCamelCase   ):



       def __init__(				self     ,UpperCAmelCase_      )  ->							Any:
           lowercase__		       =						bos_token_id







       def __call__(				self     ,UpperCAmelCase_     ,UpperCAmelCase_     ,UpperCAmelCase_      )  ->							jnp.ndarray:
           lowercase__		       =						jnp.full(scores.shape     ,-float("inf"      )      )

           lowercase__		       =						1 - jnp.bool_(cur_len - 1      )

           lowercase__		       =						jnp.where(UpperCAmelCase_     ,new_scores.at[:, self.bos_token_id].set(0      )     ,UpperCAmelCase_      )

           return scores





class 				snake_case (UpperCamelCase   ):



       def __init__(				self     ,UpperCAmelCase_     ,UpperCAmelCase_      )  ->							Any:
           lowercase__		       =						max_length
           lowercase__		       =						eos_token_id







       def __call__(				self     ,UpperCAmelCase_     ,UpperCAmelCase_     ,UpperCAmelCase_      )  ->							jnp.ndarray:
           lowercase__		       =						jnp.full(scores.shape     ,-float("inf"      )      )

           lowercase__		       =						1 - jnp.bool_(cur_len - self.max_length + 1      )

           lowercase__		       =						jnp.where(UpperCAmelCase_     ,new_scores.at[:, self.eos_token_id].set(0      )     ,UpperCAmelCase_      )

           return scores





class 				snake_case (UpperCamelCase   ):



       def __init__(				self     ,UpperCAmelCase_     ,UpperCAmelCase_      )  ->							Optional[Any]:
           if not isinstance(UpperCAmelCase_     ,UpperCAmelCase_      ) or min_length < 0:
               raise ValueError(F'''`min_length` has to be a positive integer, but is {min_length}'''      )

           if not isinstance(UpperCAmelCase_     ,UpperCAmelCase_      ) or eos_token_id < 0:
               raise ValueError(F'''`eos_token_id` has to be a positive integer, but is {eos_token_id}'''      )

           lowercase__		       =						min_length
           lowercase__		       =						eos_token_id







       def __call__(				self     ,UpperCAmelCase_     ,UpperCAmelCase_     ,UpperCAmelCase_      )  ->							jnp.ndarray:
           # create boolean flag to decide if min length penalty should be applied
           lowercase__		       =						1 - jnp.clip(cur_len - self.min_length     ,0     ,1      )

           lowercase__		       =						jnp.where(UpperCAmelCase_     ,scores.at[:, self.eos_token_id].set(-float("inf"      )      )     ,UpperCAmelCase_      )

           return scores





class 				snake_case (UpperCamelCase   ):



       def __init__(				self     ,UpperCAmelCase_     ,UpperCAmelCase_      )  ->							Any:
           lowercase__		       =						list(UpperCAmelCase_      )
           lowercase__		       =						begin_index







       def __call__(				self     ,UpperCAmelCase_     ,UpperCAmelCase_     ,UpperCAmelCase_      )  ->							Any:
           lowercase__		       =						1 - jnp.bool_(cur_len - self.begin_index      )

           lowercase__		       =						jnp.where(UpperCAmelCase_     ,scores.at[:, self.begin_suppress_tokens].set(-float("inf"      )      )     ,UpperCAmelCase_      )

           return scores





class 				snake_case (UpperCamelCase   ):



       def __init__(				self     ,UpperCAmelCase_      )  ->							Any:
           lowercase__		       =						list(UpperCAmelCase_      )







       def __call__(				self     ,UpperCAmelCase_     ,UpperCAmelCase_     ,UpperCAmelCase_      )  ->							jnp.ndarray:
           lowercase__		       =						scores.at[..., self.suppress_tokens].set(-float("inf"      )      )

           return scores





class 				snake_case (UpperCamelCase   ):



       def __init__(				self     ,UpperCAmelCase_      )  ->							Dict:
           lowercase__		       =						dict(UpperCAmelCase_      )
           # Converts the dictionary of format {index: token} containing the tokens to be forced to an array, where the
           # index of the array corresponds to the index of the token to be forced, for XLA compatibility.
           # Indexes without forced tokens will have a negative value.
           lowercase__		       =						jnp.ones((max(force_token_map.keys()      ) + 1)     ,dtype=jnp.intaa      ) * -1
           for index, token in force_token_map.items():
               if token is not None:
                   lowercase__		       =						force_token_array.at[index].set(UpperCAmelCase_      )
           lowercase__		       =						jnp.intaa(UpperCAmelCase_      )







       def __call__(				self     ,UpperCAmelCase_     ,UpperCAmelCase_     ,UpperCAmelCase_      )  ->							jnp.ndarray:
           def _force_token(UpperCAmelCase_      ):
               lowercase__		       =						scores.shape[0]
               lowercase__		       =						self.force_token_array[generation_idx]

               lowercase__		       =						jnp.ones_like(UpperCAmelCase_     ,dtype=scores.dtype      ) * -float("inf"      )
               lowercase__		       =						jnp.zeros((batch_size, 1)     ,dtype=scores.dtype      )
               lowercase__		       =						lax.dynamic_update_slice(UpperCAmelCase_     ,UpperCAmelCase_     ,(0, current_token)      )
               return new_scores

           lowercase__		       =						lax.cond(
               cur_len >= self.force_token_array.shape[0]     ,lambda: scores     ,lambda: lax.cond(
                   self.force_token_array[cur_len] >= 0     ,lambda: _force_token(UpperCAmelCase_      )     ,lambda: scores     ,)     ,)
           return scores





class 				snake_case (UpperCamelCase   ):



       def __init__(				self     ,UpperCAmelCase_     ,UpperCAmelCase_     ,UpperCAmelCase_      )  ->							Union[str, Any]:
           lowercase__		       =						generate_config.eos_token_id
           lowercase__		       =						generate_config.no_timestamps_token_id
           lowercase__		       =						generate_config.no_timestamps_token_id + 1

           lowercase__		       =						decoder_input_length + 1

           if generate_config.is_multilingual:
               # room for language token and task token
               self.begin_index += 2
           if hasattr(UpperCAmelCase_     ,"max_initial_timestamp_index"      ):
               lowercase__		       =						generate_config.max_initial_timestamp_index
           else:
               lowercase__		       =						model_config.vocab_size
           if self.max_initial_timestamp_index is None:
               lowercase__		       =						model_config.vocab_size







       def __call__(				self     ,UpperCAmelCase_     ,UpperCAmelCase_     ,UpperCAmelCase_      )  ->							int:
           # suppress <|notimestamps|> which is handled by without_timestamps
           lowercase__		       =						scores.at[:, self.no_timestamps_token_id].set(-float("inf"      )      )

           def handle_pairs(UpperCAmelCase_     ,UpperCAmelCase_      ):
               lowercase__		       =						jnp.where((cur_len - self.begin_index) >= 1     ,UpperCAmelCase_     ,UpperCAmelCase_      )
               lowercase__		       =						jnp.where(
                   input_ids_k[cur_len - 1] >= self.timestamp_begin     ,True and last_was_timestamp     ,UpperCAmelCase_     ,)

               lowercase__		       =						jnp.where((cur_len - self.begin_index) < 2     ,UpperCAmelCase_     ,UpperCAmelCase_      )
               lowercase__		       =						jnp.where(
                   input_ids_k[cur_len - 2] >= self.timestamp_begin     ,UpperCAmelCase_     ,UpperCAmelCase_     ,)

               return jnp.where(
                   UpperCAmelCase_     ,jnp.where(
                       penultimate_was_timestamp > 0     ,scores_k.at[self.timestamp_begin :].set(-float("inf"      )      )     ,scores_k.at[: self.eos_token_id].set(-float("inf"      )      )     ,)     ,UpperCAmelCase_     ,)

           lowercase__		       =						jax.vmap(UpperCAmelCase_      )(UpperCAmelCase_     ,UpperCAmelCase_      )

           lowercase__		       =						jnp.where(cur_len == self.begin_index     ,UpperCAmelCase_     ,UpperCAmelCase_      )
           lowercase__		       =						jnp.where(
               self.max_initial_timestamp_index is not None     ,True and apply_max_initial_timestamp     ,UpperCAmelCase_     ,)

           lowercase__		       =						self.timestamp_begin + self.max_initial_timestamp_index

           lowercase__		       =						jnp.where(
               UpperCAmelCase_     ,scores.at[:, last_allowed + 1 :].set(-float("inf"      )      )     ,UpperCAmelCase_     ,)

           # if sum of probability over timestamps is above any other token, sample timestamp
           lowercase__		       =						jax.nn.log_softmax(UpperCAmelCase_     ,axis=-1      )

           def handle_cumulative_probs(UpperCAmelCase_     ,UpperCAmelCase_      ):
               lowercase__		       =						jax.nn.logsumexp(logprobs_k[self.timestamp_begin :]     ,axis=-1      )
               lowercase__		       =						jnp.max(logprobs_k[: self.timestamp_begin]      )
               return jnp.where(
                   timestamp_logprob > max_text_token_logprob     ,scores_k.at[: self.timestamp_begin].set(-float("inf"      )      )     ,UpperCAmelCase_     ,)

           lowercase__		       =						jax.vmap(UpperCAmelCase_      )(UpperCAmelCase_     ,UpperCAmelCase_      )

           return scores









'''simple docstring'''





import copy
import random

from transformers import CLIPTokenizer


class 				snake_case (UpperCamelCase   ):



       def __init__(				self     ,*UpperCAmelCase_     ,**UpperCAmelCase_      )  ->							List[str]:
           super().__init__(*UpperCAmelCase_     ,**UpperCAmelCase_      )
           lowercase__		       =						{}



       def 							_a      (				self     ,UpperCAmelCase_     ,*UpperCAmelCase_     ,**UpperCAmelCase_      )  ->							str:
           lowercase__		       =						super().add_tokens(UpperCAmelCase_     ,*UpperCAmelCase_     ,**UpperCAmelCase_      )
           if num_added_tokens == 0:
               raise ValueError(
                   F'''The tokenizer already contains the token {placeholder_token}. Please pass a different'''
                   " `placeholder_token` that is not already in the tokenizer."      )



       def 							_a      (				self     ,UpperCAmelCase_     ,*UpperCAmelCase_     ,UpperCAmelCase_=1     ,**UpperCAmelCase_      )  ->							List[Any]:
           lowercase__		       =						[]
           if num_vec_per_token == 1:
               self.try_adding_tokens(UpperCAmelCase_     ,*UpperCAmelCase_     ,**UpperCAmelCase_      )
               output.append(UpperCAmelCase_      )
           else:
               lowercase__		       =						[]
               for i in range(UpperCAmelCase_      ):
                   lowercase__		       =						placeholder_token + F'''_{i}'''
                   self.try_adding_tokens(UpperCAmelCase_     ,*UpperCAmelCase_     ,**UpperCAmelCase_      )
                   output.append(UpperCAmelCase_      )
        # handle cases where there is a new placeholder token that contains the current placeholder token but is larger
           for token in self.token_map:
               if token in placeholder_token:
                   raise ValueError(
                       F'''The tokenizer already has placeholder token {token} that can get confused with'''
                       F''' {placeholder_token}keep placeholder tokens independent'''      )
           lowercase__		       =						output



       def 							_a      (				self     ,UpperCAmelCase_     ,UpperCAmelCase_=False     ,UpperCAmelCase_=1.0      )  ->							Optional[int]:
           if isinstance(UpperCAmelCase_     ,UpperCAmelCase_      ):
               lowercase__		       =						[]
               for i in range(len(UpperCAmelCase_      )      ):
                   output.append(self.replace_placeholder_tokens_in_text(text[i]     ,vector_shuffle=UpperCAmelCase_      )      )
               return output
           for placeholder_token in self.token_map:
               if placeholder_token in text:
                   lowercase__		       =						self.token_map[placeholder_token]
                   lowercase__		       =						tokens[: 1 + int(len(UpperCAmelCase_      ) * prop_tokens_to_load      )]
                   if vector_shuffle:
                       lowercase__		       =						copy.copy(UpperCAmelCase_      )
                       random.shuffle(UpperCAmelCase_      )
                   lowercase__		       =						text.replace(UpperCAmelCase_     ," ".join(UpperCAmelCase_      )      )
           return text



       def __call__(				self     ,UpperCAmelCase_     ,*UpperCAmelCase_     ,UpperCAmelCase_=False     ,UpperCAmelCase_=1.0     ,**UpperCAmelCase_      )  ->							int:
           return super().__call__(
               self.replace_placeholder_tokens_in_text(
                   UpperCAmelCase_     ,vector_shuffle=UpperCAmelCase_     ,prop_tokens_to_load=UpperCAmelCase_      )     ,*UpperCAmelCase_     ,**UpperCAmelCase_     ,)







       def 							_a      (				self     ,UpperCAmelCase_     ,*UpperCAmelCase_     ,UpperCAmelCase_=False     ,UpperCAmelCase_=1.0     ,**UpperCAmelCase_      )  ->							Optional[int]:
           return super().encode(
               self.replace_placeholder_tokens_in_text(
                   UpperCAmelCase_     ,vector_shuffle=UpperCAmelCase_     ,prop_tokens_to_load=UpperCAmelCase_      )     ,*UpperCAmelCase_     ,**UpperCAmelCase_     ,)




