
import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, BatchEncoding, PreTrainedTokenizer
from ...utils import logging


_A     =  logging.get_logger(__name__)

_A     =  "▁"

_A     =  {"vocab_file": "sentencepiece.bpe.model"}

_A     =  {
    "vocab_file": {
        "facebook/mbart-large-50-one-to-many-mmt": (
            "https://huggingface.co/facebook/mbart-large-50-one-to-many-mmt/resolve/main/sentencepiece.bpe.model"
        ),
    }
}

_A     =  {
    "facebook/mbart-large-50-one-to-many-mmt": 10_24,
}

# fmt: off
_A     =  ["ar_AR", "cs_CZ", "de_DE", "en_XX", "es_XX", "et_EE", "fi_FI", "fr_XX", "gu_IN", "hi_IN", "it_IT", "ja_XX", "kk_KZ", "ko_KR", "lt_LT", "lv_LV", "my_MM", "ne_NP", "nl_XX", "ro_RO", "ru_RU", "si_LK", "tr_TR", "vi_VN", "zh_CN", "af_ZA", "az_AZ", "bn_IN", "fa_IR", "he_IL", "hr_HR", "id_ID", "ka_GE", "km_KH", "mk_MK", "ml_IN", "mn_MN", "mr_IN", "pl_PL", "ps_AF", "pt_XX", "sv_SE", "sw_KE", "ta_IN", "te_IN", "th_TH", "tl_XX", "uk_UA", "ur_PK", "xh_ZA", "gl_ES", "sl_SI"]





class 					_lowerCAmelCase      (			__lowercase ):
				_lowercase    	=VOCAB_FILES_NAMES
				_lowercase    	=PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				_lowercase    	=PRETRAINED_VOCAB_FILES_MAP
				_lowercase    	=['''input_ids''', '''attention_mask''']

				_lowercase    	=[]
				_lowercase    	=[]




				def __init__(      self       ,       _UpperCamelCase       ,       _UpperCamelCase=None       ,       _UpperCamelCase=None       ,       _UpperCamelCase="</s>"       ,       _UpperCamelCase="</s>"       ,       _UpperCamelCase="<s>"       ,       _UpperCamelCase="<unk>"       ,       _UpperCamelCase="<pad>"       ,       _UpperCamelCase="<mask>"       ,       _UpperCamelCase = None       ,       **_UpperCamelCase       ,       )       ->   Union[str, Any]:
							# Mask token behave like a normal word, i.e. include the space before it
							lowerCAmelCase_      			=			AddedToken(_A       ,       lstrip=_A       ,       rstrip=_A  ) if isinstance(_A       ,       _A  ) else mask_token

							lowerCAmelCase_      			=			{} if sp_model_kwargs is None else sp_model_kwargs

							lowerCAmelCase_      			=			kwargs.get("additional_special_tokens"       ,       []  )
							kwargs["additional_special_tokens"] += [
							    code for code in FAIRSEQ_LANGUAGE_CODES if code not in kwargs["additional_special_tokens"]
							]

							super().__init__(
							    src_lang=_A       ,       tgt_lang=_A       ,       eos_token=_A       ,       unk_token=_A       ,       sep_token=_A       ,       cls_token=_A       ,       pad_token=_A       ,       mask_token=_A       ,       sp_model_kwargs=self.sp_model_kwargs       ,       **_A       ,       )

							lowerCAmelCase_      			=			spm.SentencePieceProcessor(**self.sp_model_kwargs  )
							self.sp_model.Load(str(_A  )  )
							lowerCAmelCase_      			=			vocab_file

							# Original fairseq vocab and spm vocab must be "aligned":
							# Vocab    |    0    |    1    |   2    |    3    |  4  |  5  |  6  |   7   |   8   |  9
							# -------- | ------- | ------- | ------ | ------- | --- | --- | --- | ----- | ----- | ----
							# fairseq  | '<s>'   | '<pad>' | '</s>' | '<unk>' | ',' | '.' | '▁' | 's'   | '▁de' | '-'
							# spm      | '<unk>' | '<s>'   | '</s>' | ','     | '.' | '▁' | 's' | '▁de' | '-'   | '▁a'

							# Mimic fairseq token-to-id alignment for the first 4 token
							lowerCAmelCase_      			=			{"<s>": 0, "<pad>": 1, "</s>": 2, "<unk>": 3}

							# The first "real" token "," has position 4 in the original fairseq vocab and position 3 in the spm vocab
							lowerCAmelCase_      			=			1

							lowerCAmelCase_      			=			len(self.sp_model  )
							lowerCAmelCase_      			=			{
							    code: self.sp_model_size + i + self.fairseq_offset for i, code in enumerate(_A  )
							}
							lowerCAmelCase_      			=			{v: k for k, v in self.lang_code_to_id.items()}
							lowerCAmelCase_      			=			len(self.sp_model  ) + len(self.lang_code_to_id  ) + self.fairseq_offset

							self.fairseq_tokens_to_ids.update(self.lang_code_to_id  )
							lowerCAmelCase_      			=			{v: k for k, v in self.fairseq_tokens_to_ids.items()}

							lowerCAmelCase_      			=			src_lang if src_lang is not None else "en_XX"
							lowerCAmelCase_      			=			self.lang_code_to_id[self._src_lang]
							lowerCAmelCase_      			=			tgt_lang
							self.set_src_lang_special_tokens(self._src_lang  )




				@property
				def   __a		(      self  )       ->   Tuple:
							return len(self.sp_model  ) + len(self.lang_code_to_id  ) + self.fairseq_offset + 1  # Plus 1 for the mask token




				@property
				def   __a		(      self  )       ->   List[Any]:
							return self._src_lang




				@src_lang.setter
				def   __a		(      self       ,       _UpperCamelCase  )       ->   List[str]:
							lowerCAmelCase_      			=			new_src_lang
							self.set_src_lang_special_tokens(self._src_lang  )




				def __getstate__(      self  )       ->   Tuple:
							lowerCAmelCase_      			=			self.__dict__.copy()
							lowerCAmelCase_      			=			None
							return state




				def __setstate__(      self       ,       _UpperCamelCase  )       ->   Any:
							lowerCAmelCase_      			=			d

							# for backward compatibility
							if not hasattr(self       ,       "sp_model_kwargs"  ):
										lowerCAmelCase_      			=			{}

							lowerCAmelCase_      			=			spm.SentencePieceProcessor(**self.sp_model_kwargs  )
							self.sp_model.Load(self.vocab_file  )




				def   __a		(      self  )       ->   Any:
							lowerCAmelCase_      			=			{self.convert_ids_to_tokens(_A  ): i for i in range(self.vocab_size  )}
							vocab.update(self.added_tokens_encoder  )
							return vocab




				def   __a		(      self       ,       _UpperCamelCase  )       ->   Optional[int]:
							return self.sp_model.encode(_A       ,       out_type=_A  )




				def   __a		(      self       ,       _UpperCamelCase  )       ->   Dict:
							if token in self.fairseq_tokens_to_ids:
										return self.fairseq_tokens_to_ids[token]
							lowerCAmelCase_      			=			self.sp_model.PieceToId(_A  )

							# Need to return unknown token if the SP model returned 0
							return spm_id + self.fairseq_offset if spm_id else self.unk_token_id




				def   __a		(      self       ,       _UpperCamelCase  )       ->   List[Any]:
							if index in self.fairseq_ids_to_tokens:
										return self.fairseq_ids_to_tokens[index]
							return self.sp_model.IdToPiece(index - self.fairseq_offset  )




				def   __a		(      self       ,       _UpperCamelCase  )       ->   List[Any]:
							lowerCAmelCase_      			=			[]
							lowerCAmelCase_      			=			""
							lowerCAmelCase_      			=			False
							for token in tokens:
										# make sure that special tokens are not decoded using sentencepiece model
										if token in self.all_special_tokens:
													if not prev_is_special:
																out_string += " "
													out_string += self.sp_model.decode(_A  ) + token
													lowerCAmelCase_      			=			True
													lowerCAmelCase_      			=			[]
										else:
													current_sub_tokens.append(_A  )
													lowerCAmelCase_      			=			False
							out_string += self.sp_model.decode(_A  )
							return out_string.strip()




				def   __a		(      self       ,       _UpperCamelCase       ,       _UpperCamelCase = None  )       ->   int:
							if not os.path.isdir(_A  ):
										logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""  )
										return
							lowerCAmelCase_      			=			os.path.join(
							    _A       ,       (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]  )

							if os.path.abspath(self.vocab_file  ) != os.path.abspath(_A  ) and os.path.isfile(self.vocab_file  ):
										copyfile(self.vocab_file       ,       _A  )
							elif not os.path.isfile(self.vocab_file  ):
										with open(_A       ,       "wb"  ) as fi:
													lowerCAmelCase_      			=			self.sp_model.serialized_model_proto()
													fi.write(_A  )

							return (out_vocab_file,)




				def   __a		(      self       ,       _UpperCamelCase       ,       _UpperCamelCase = None       ,       _UpperCamelCase = False  )       ->   List[Any]:

							if already_has_special_tokens:
										return super().get_special_tokens_mask(
										    token_ids_a=_A       ,       token_ids_a=_A       ,       already_has_special_tokens=_A  )

							lowerCAmelCase_      			=			[1] * len(self.prefix_tokens  )
							lowerCAmelCase_      			=			[1] * len(self.suffix_tokens  )
							if token_ids_a is None:
										return prefix_ones + ([0] * len(_A  )) + suffix_ones
							return prefix_ones + ([0] * len(_A  )) + ([0] * len(_A  )) + suffix_ones




				def   __a		(      self       ,       _UpperCamelCase       ,       _UpperCamelCase = None  )       ->   Optional[Any]:
							if token_ids_a is None:
										return self.prefix_tokens + token_ids_a + self.suffix_tokens
							# We don't expect to process pairs, but leave the pair logic for API consistency
							return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens




				def   __a		(      self       ,       _UpperCamelCase       ,       _UpperCamelCase       ,       _UpperCamelCase       ,       _UpperCamelCase       ,       **_UpperCamelCase  )       ->   str:
							if src_lang is None or tgt_lang is None:
										raise ValueError("Translation requires a `src_lang` and a `tgt_lang` for this model"  )
							lowerCAmelCase_      			=			src_lang
							lowerCAmelCase_      			=			self(_A       ,       add_special_tokens=_A       ,       return_tensors=_A       ,       **_A  )
							lowerCAmelCase_      			=			self.convert_tokens_to_ids(_A  )
							lowerCAmelCase_      			=			tgt_lang_id
							return inputs




				def   __a		(      self       ,       _UpperCamelCase       ,       _UpperCamelCase = "en_XX"       ,       _UpperCamelCase = None       ,       _UpperCamelCase = "ro_RO"       ,       **_UpperCamelCase       ,       )       ->   str:
							lowerCAmelCase_      			=			src_lang
							lowerCAmelCase_      			=			tgt_lang
							return super().prepare_seqaseq_batch(_A       ,       _A       ,       **_A  )




				def   __a		(      self  )       ->   Optional[Any]:
							return self.set_src_lang_special_tokens(self.src_lang  )




				def   __a		(      self  )       ->   List[Any]:
							return self.set_tgt_lang_special_tokens(self.tgt_lang  )




				def   __a		(      self       ,       _UpperCamelCase  )       ->   Any:
							lowerCAmelCase_      			=			self.lang_code_to_id[src_lang]
							lowerCAmelCase_      			=			[self.cur_lang_code_id]
							lowerCAmelCase_      			=			[self.eos_token_id]




				def   __a		(      self       ,       _UpperCamelCase  )       ->   Tuple:
							lowerCAmelCase_      			=			self.lang_code_to_id[tgt_lang]
							lowerCAmelCase_      			=			[self.cur_lang_code_id]
							lowerCAmelCase_      			=			[self.eos_token_id]





import inspect

import jax
import jax.lax as lax
import jax.numpy as jnp

from ..utils import add_start_docstrings
from ..utils.logging import get_logger


_A     =  get_logger(__name__)


_A     =  R"\n    Args:\n        input_ids (`jnp.ndarray` of shape `(batch_size, sequence_length)`):\n            Indices of input sequence tokens in the vocabulary.\n\n            Indices can be obtained using [`PreTrainedTokenizer`]. See [`PreTrainedTokenizer.encode`] and\n            [`PreTrainedTokenizer.__call__`] for details.\n\n            [What are input IDs?](../glossary#input-ids)\n        scores (`jnp.ndarray` of shape `(batch_size, config.vocab_size)`):\n            Prediction scores of a language modeling head. These can be logits for each vocabulary when not using beam\n            search or log softmax for each vocabulary token when using beam search\n        kwargs (`Dict[str, Any]`, *optional*):\n            Additional logits processor specific kwargs.\n\n    Return:\n        `jnp.ndarray` of shape `(batch_size, config.vocab_size)`: The processed prediction scores.\n\n"





class 			_lowerCAmelCase      :




				@add_start_docstrings(_UpperCamelCase  )
				def __call__(      self       ,       _UpperCamelCase       ,       _UpperCamelCase  )       ->   jnp.ndarray:
							raise NotImplementedError(
							    f"""{self.__class__} is an abstract class. Only classes inheriting this class can be called."""  )





class 			_lowerCAmelCase      :




				@add_start_docstrings(_UpperCamelCase  )
				def __call__(      self       ,       _UpperCamelCase       ,       _UpperCamelCase  )       ->   jnp.ndarray:
							raise NotImplementedError(
							    f"""{self.__class__} is an abstract class. Only classes inheriting this class can be called."""  )





class 			_lowerCAmelCase      (			__a ):




				@add_start_docstrings(_UpperCamelCase  )
				def __call__(      self       ,       _UpperCamelCase       ,       _UpperCamelCase       ,       _UpperCamelCase       ,       **_UpperCamelCase  )       ->   jnp.ndarray:
							for processor in self:
										lowerCAmelCase_      			=			inspect.signature(processor.__call__  ).parameters
										if len(_UpperCamelCase  ) > 3:
													if not all(arg in kwargs for arg in list(function_args.keys()  )[2:]  ):
																raise ValueError(
																    f"""Make sure that all the required parameters: {list(function_args.keys()  )} for """
																    f"""{processor.__class__} are passed to the logits processor."""  )
													lowerCAmelCase_      			=			processor(_UpperCamelCase       ,       _UpperCamelCase       ,       _UpperCamelCase       ,       **_UpperCamelCase  )
										else:
													lowerCAmelCase_      			=			processor(_UpperCamelCase       ,       _UpperCamelCase       ,       _UpperCamelCase  )
							return scores





class 			_lowerCAmelCase      (			__a ):




				def __init__(      self       ,       _UpperCamelCase  )       ->   Tuple:
							if not isinstance(_UpperCamelCase       ,       _UpperCamelCase  ) or not (temperature > 0):
										raise ValueError(f"""`temperature` has to be a strictly positive float, but is {temperature}"""  )

							lowerCAmelCase_      			=			temperature




				def __call__(      self       ,       _UpperCamelCase       ,       _UpperCamelCase       ,       _UpperCamelCase  )       ->   jnp.ndarray:
							lowerCAmelCase_      			=			scores / self.temperature
							return scores





class 			_lowerCAmelCase      (			__a ):




				def __init__(      self       ,       _UpperCamelCase       ,       _UpperCamelCase = -float("Inf"  )       ,       _UpperCamelCase = 1  )       ->   Union[str, Any]:
							if not isinstance(_UpperCamelCase       ,       _UpperCamelCase  ) or (top_p < 0 or top_p > 1.0):
										raise ValueError(f"""`top_p` has to be a float > 0 and < 1, but is {top_p}"""  )
							if not isinstance(_UpperCamelCase       ,       _UpperCamelCase  ) or (min_tokens_to_keep < 1):
										raise ValueError(f"""`min_tokens_to_keep` has to be a positive integer, but is {min_tokens_to_keep}"""  )

							lowerCAmelCase_      			=			top_p
							lowerCAmelCase_      			=			filter_value
							lowerCAmelCase_      			=			min_tokens_to_keep




				def __call__(      self       ,       _UpperCamelCase       ,       _UpperCamelCase       ,       _UpperCamelCase  )       ->   jnp.ndarray:
							lowerCAmelCase_			,						lowerCAmelCase_      			=			lax.top_k(_UpperCamelCase       ,       scores.shape[-1]  )

							lowerCAmelCase_      			=			jnp.full_like(_UpperCamelCase       ,       self.filter_value  )
							lowerCAmelCase_      			=			jax.nn.softmax(_UpperCamelCase       ,       axis=-1  ).cumsum(axis=-1  )
							lowerCAmelCase_      			=			cumulative_probs < self.top_p

							# include the token that is higher than top_p as well
							lowerCAmelCase_      			=			jnp.roll(_UpperCamelCase       ,       1  )
							score_mask |= score_mask.at[:, 0].set(_UpperCamelCase  )

							# min tokens to keep
							lowerCAmelCase_      			=			score_mask.at[:, : self.min_tokens_to_keep].set(_UpperCamelCase  )

							lowerCAmelCase_      			=			jnp.where(_UpperCamelCase       ,       _UpperCamelCase       ,       _UpperCamelCase  )
							lowerCAmelCase_      			=			jax.lax.sort_key_val(_UpperCamelCase       ,       _UpperCamelCase  )[-1]

							return next_scores





class 			_lowerCAmelCase      (			__a ):




				def __init__(      self       ,       _UpperCamelCase       ,       _UpperCamelCase = -float("Inf"  )       ,       _UpperCamelCase = 1  )       ->   List[Any]:
							if not isinstance(_UpperCamelCase       ,       _UpperCamelCase  ) or top_k <= 0:
										raise ValueError(f"""`top_k` has to be a strictly positive integer, but is {top_k}"""  )

							lowerCAmelCase_      			=			max(_UpperCamelCase       ,       _UpperCamelCase  )
							lowerCAmelCase_      			=			filter_value




				def __call__(      self       ,       _UpperCamelCase       ,       _UpperCamelCase       ,       _UpperCamelCase  )       ->   jnp.ndarray:
							lowerCAmelCase_			,						lowerCAmelCase_      			=			scores.shape
							lowerCAmelCase_      			=			jnp.full(batch_size * vocab_size       ,       self.filter_value  )

							lowerCAmelCase_      			=			min(self.top_k       ,       scores.shape[-1]  )  # Safety check
							lowerCAmelCase_			,						lowerCAmelCase_      			=			lax.top_k(_UpperCamelCase       ,       _UpperCamelCase  )
							lowerCAmelCase_      			=			jnp.broadcast_to((jnp.arange(_UpperCamelCase  ) * vocab_size)[:, None]       ,       (batch_size, topk)  ).flatten()
							lowerCAmelCase_      			=			topk_scores.flatten()
							lowerCAmelCase_      			=			topk_indices.flatten() + shift

							lowerCAmelCase_      			=			next_scores_flat.at[topk_indices_flat].set(_UpperCamelCase  )
							lowerCAmelCase_      			=			next_scores_flat.reshape(_UpperCamelCase       ,       _UpperCamelCase  )
							return next_scores





class 			_lowerCAmelCase      (			__a ):




				def __init__(      self       ,       _UpperCamelCase  )       ->   Any:
							lowerCAmelCase_      			=			bos_token_id




				def __call__(      self       ,       _UpperCamelCase       ,       _UpperCamelCase       ,       _UpperCamelCase  )       ->   jnp.ndarray:
							lowerCAmelCase_      			=			jnp.full(scores.shape       ,       -float("inf"  )  )

							lowerCAmelCase_      			=			1 - jnp.bool_(cur_len - 1  )

							lowerCAmelCase_      			=			jnp.where(_UpperCamelCase       ,       new_scores.at[:, self.bos_token_id].set(0  )       ,       _UpperCamelCase  )

							return scores





class 			_lowerCAmelCase      (			__a ):




				def __init__(      self       ,       _UpperCamelCase       ,       _UpperCamelCase  )       ->   List[str]:
							lowerCAmelCase_      			=			max_length
							lowerCAmelCase_      			=			eos_token_id




				def __call__(      self       ,       _UpperCamelCase       ,       _UpperCamelCase       ,       _UpperCamelCase  )       ->   jnp.ndarray:
							lowerCAmelCase_      			=			jnp.full(scores.shape       ,       -float("inf"  )  )

							lowerCAmelCase_      			=			1 - jnp.bool_(cur_len - self.max_length + 1  )

							lowerCAmelCase_      			=			jnp.where(_UpperCamelCase       ,       new_scores.at[:, self.eos_token_id].set(0  )       ,       _UpperCamelCase  )

							return scores





class 			_lowerCAmelCase      (			__a ):




				def __init__(      self       ,       _UpperCamelCase       ,       _UpperCamelCase  )       ->   Optional[int]:
							if not isinstance(_UpperCamelCase       ,       _UpperCamelCase  ) or min_length < 0:
										raise ValueError(f"""`min_length` has to be a positive integer, but is {min_length}"""  )

							if not isinstance(_UpperCamelCase       ,       _UpperCamelCase  ) or eos_token_id < 0:
										raise ValueError(f"""`eos_token_id` has to be a positive integer, but is {eos_token_id}"""  )

							lowerCAmelCase_      			=			min_length
							lowerCAmelCase_      			=			eos_token_id




				def __call__(      self       ,       _UpperCamelCase       ,       _UpperCamelCase       ,       _UpperCamelCase  )       ->   jnp.ndarray:
							# create boolean flag to decide if min length penalty should be applied
							lowerCAmelCase_      			=			1 - jnp.clip(cur_len - self.min_length       ,       0       ,       1  )

							lowerCAmelCase_      			=			jnp.where(_UpperCamelCase       ,       scores.at[:, self.eos_token_id].set(-float("inf"  )  )       ,       _UpperCamelCase  )

							return scores





class 			_lowerCAmelCase      (			__a ):




				def __init__(      self       ,       _UpperCamelCase       ,       _UpperCamelCase  )       ->   int:
							lowerCAmelCase_      			=			list(_UpperCamelCase  )
							lowerCAmelCase_      			=			begin_index




				def __call__(      self       ,       _UpperCamelCase       ,       _UpperCamelCase       ,       _UpperCamelCase  )       ->   int:
							lowerCAmelCase_      			=			1 - jnp.bool_(cur_len - self.begin_index  )

							lowerCAmelCase_      			=			jnp.where(_UpperCamelCase       ,       scores.at[:, self.begin_suppress_tokens].set(-float("inf"  )  )       ,       _UpperCamelCase  )

							return scores





class 			_lowerCAmelCase      (			__a ):




				def __init__(      self       ,       _UpperCamelCase  )       ->   Optional[Any]:
							lowerCAmelCase_      			=			list(_UpperCamelCase  )




				def __call__(      self       ,       _UpperCamelCase       ,       _UpperCamelCase       ,       _UpperCamelCase  )       ->   jnp.ndarray:
							lowerCAmelCase_      			=			scores.at[..., self.suppress_tokens].set(-float("inf"  )  )

							return scores





class 			_lowerCAmelCase      (			__a ):




				def __init__(      self       ,       _UpperCamelCase  )       ->   List[Any]:
							lowerCAmelCase_      			=			dict(_UpperCamelCase  )
							# Converts the dictionary of format {index: token} containing the tokens to be forced to an array, where the
							# index of the array corresponds to the index of the token to be forced, for XLA compatibility.
							# Indexes without forced tokens will have a negative value.
							lowerCAmelCase_      			=			jnp.ones((max(force_token_map.keys()  ) + 1)       ,       dtype=jnp.intaa  ) * -1
							for index, token in force_token_map.items():
										if token is not None:
													lowerCAmelCase_      			=			force_token_array.at[index].set(_UpperCamelCase  )
							lowerCAmelCase_      			=			jnp.intaa(_UpperCamelCase  )




				def __call__(      self       ,       _UpperCamelCase       ,       _UpperCamelCase       ,       _UpperCamelCase  )       ->   jnp.ndarray:
							def _force_token(_UpperCamelCase  ):
										lowerCAmelCase_      			=			scores.shape[0]
										lowerCAmelCase_      			=			self.force_token_array[generation_idx]

										lowerCAmelCase_      			=			jnp.ones_like(_UpperCamelCase       ,       dtype=scores.dtype  ) * -float("inf"  )
										lowerCAmelCase_      			=			jnp.zeros((batch_size, 1)       ,       dtype=scores.dtype  )
										lowerCAmelCase_      			=			lax.dynamic_update_slice(_UpperCamelCase       ,       _UpperCamelCase       ,       (0, current_token)  )
										return new_scores

							lowerCAmelCase_      			=			lax.cond(
							    cur_len >= self.force_token_array.shape[0]       ,       lambda: scores       ,       lambda: lax.cond(
							        self.force_token_array[cur_len] >= 0       ,       lambda: _force_token(_UpperCamelCase  )       ,       lambda: scores       ,       )       ,       )
							return scores





class 			_lowerCAmelCase      (			__a ):




				def __init__(      self       ,       _UpperCamelCase       ,       _UpperCamelCase       ,       _UpperCamelCase  )       ->   Tuple:
							lowerCAmelCase_      			=			generate_config.eos_token_id
							lowerCAmelCase_      			=			generate_config.no_timestamps_token_id
							lowerCAmelCase_      			=			generate_config.no_timestamps_token_id + 1

							lowerCAmelCase_      			=			decoder_input_length + 1

							if generate_config.is_multilingual:
										# room for language token and task token
										self.begin_index += 2
							if hasattr(_UpperCamelCase       ,       "max_initial_timestamp_index"  ):
										lowerCAmelCase_      			=			generate_config.max_initial_timestamp_index
							else:
										lowerCAmelCase_      			=			model_config.vocab_size
							if self.max_initial_timestamp_index is None:
										lowerCAmelCase_      			=			model_config.vocab_size




				def __call__(      self       ,       _UpperCamelCase       ,       _UpperCamelCase       ,       _UpperCamelCase  )       ->   Optional[Any]:
							# suppress <|notimestamps|> which is handled by without_timestamps
							lowerCAmelCase_      			=			scores.at[:, self.no_timestamps_token_id].set(-float("inf"  )  )

							def handle_pairs(_UpperCamelCase       ,       _UpperCamelCase  ):
										lowerCAmelCase_      			=			jnp.where((cur_len - self.begin_index) >= 1       ,       _UpperCamelCase       ,       _UpperCamelCase  )
										lowerCAmelCase_      			=			jnp.where(
										    input_ids_k[cur_len - 1] >= self.timestamp_begin       ,       True and last_was_timestamp       ,       _UpperCamelCase       ,       )

										lowerCAmelCase_      			=			jnp.where((cur_len - self.begin_index) < 2       ,       _UpperCamelCase       ,       _UpperCamelCase  )
										lowerCAmelCase_      			=			jnp.where(
										    input_ids_k[cur_len - 2] >= self.timestamp_begin       ,       _UpperCamelCase       ,       _UpperCamelCase       ,       )

										return jnp.where(
										    _UpperCamelCase       ,       jnp.where(
										        penultimate_was_timestamp > 0       ,       scores_k.at[self.timestamp_begin :].set(-float("inf"  )  )       ,       scores_k.at[: self.eos_token_id].set(-float("inf"  )  )       ,       )       ,       _UpperCamelCase       ,       )

							lowerCAmelCase_      			=			jax.vmap(_UpperCamelCase  )(_UpperCamelCase       ,       _UpperCamelCase  )

							lowerCAmelCase_      			=			jnp.where(cur_len == self.begin_index       ,       _UpperCamelCase       ,       _UpperCamelCase  )
							lowerCAmelCase_      			=			jnp.where(
							    self.max_initial_timestamp_index is not None       ,       True and apply_max_initial_timestamp       ,       _UpperCamelCase       ,       )

							lowerCAmelCase_      			=			self.timestamp_begin + self.max_initial_timestamp_index

							lowerCAmelCase_      			=			jnp.where(
							    _UpperCamelCase       ,       scores.at[:, last_allowed + 1 :].set(-float("inf"  )  )       ,       _UpperCamelCase       ,       )

							# if sum of probability over timestamps is above any other token, sample timestamp
							lowerCAmelCase_      			=			jax.nn.log_softmax(_UpperCamelCase       ,       axis=-1  )

							def handle_cumulative_probs(_UpperCamelCase       ,       _UpperCamelCase  ):
										lowerCAmelCase_      			=			jax.nn.logsumexp(logprobs_k[self.timestamp_begin :]       ,       axis=-1  )
										lowerCAmelCase_      			=			jnp.max(logprobs_k[: self.timestamp_begin]  )
										return jnp.where(
										    timestamp_logprob > max_text_token_logprob       ,       scores_k.at[: self.timestamp_begin].set(-float("inf"  )  )       ,       _UpperCamelCase       ,       )

							lowerCAmelCase_      			=			jax.vmap(_UpperCamelCase  )(_UpperCamelCase       ,       _UpperCamelCase  )

							return scores



