

'''simple docstring'''





import inspect

import jax
import jax.lax as lax
import jax.numpy as jnp

from ..utils import add_start_docstrings
from ..utils.logging import get_logger


a_						     =    get_logger(__name__)


a_						     =    r'\n    Args:\n        input_ids (`jnp.ndarray` of shape `(batch_size, sequence_length)`):\n            Indices of input sequence tokens in the vocabulary.\n\n            Indices can be obtained using [`PreTrainedTokenizer`]. See [`PreTrainedTokenizer.encode`] and\n            [`PreTrainedTokenizer.__call__`] for details.\n\n            [What are input IDs?](../glossary#input-ids)\n        scores (`jnp.ndarray` of shape `(batch_size, config.vocab_size)`):\n            Prediction scores of a language modeling head. These can be logits for each vocabulary when not using beam\n            search or log softmax for each vocabulary token when using beam search\n        kwargs (`Dict[str, Any]`, *optional*):\n            Additional logits processor specific kwargs.\n\n    Return:\n        `jnp.ndarray` of shape `(batch_size, config.vocab_size)`: The processed prediction scores.\n\n'






class _UpperCamelCase							:
			'''simple docstring'''







			@add_start_docstrings(_A	)
			def __call__(			self							: Tuple      ,      a							: Union[str, Any]      ,      a							: Union[str, Any]	)							->       Optional[Any]:

				"""simple docstring"""

				raise NotImplementedError(
				    F"{self.__class__} is an abstract class. Only classes inheriting this class can be called."	)






class _UpperCamelCase							:
			'''simple docstring'''







			@add_start_docstrings(_A	)
			def __call__(			self							: Optional[Any]      ,      a							: List[str]      ,      a							: List[str]	)							->       int:

				"""simple docstring"""

				raise NotImplementedError(
				    F"{self.__class__} is an abstract class. Only classes inheriting this class can be called."	)






class _UpperCamelCase							(							a__   ):
			'''simple docstring'''







			@add_start_docstrings(_A	)
			def __call__(			self							: Optional[int]      ,      a							: List[Any]      ,      a							: Dict      ,      a							: Optional[int]      ,      **a							: List[str]	)							->       int:

				"""simple docstring"""

				for processor in self:
					SCREAMING_SNAKE_CASE   :      List[Any]				     =       inspect.signature(processor.__call__	).parameters
					if len(_A	) > 3:
						if not all(arg in kwargs for arg in list(function_args.keys()	)[2:]	):
							raise ValueError(
							    F"Make sure that all the required parameters: {list(function_args.keys()	)} for "
							    F"{processor.__class__} are passed to the logits processor."	)
						SCREAMING_SNAKE_CASE   :      Tuple				     =       processor(_A      ,      _A      ,      _A      ,      **_A	)
					else:
						SCREAMING_SNAKE_CASE   :      Optional[int]				     =       processor(_A      ,      _A      ,      _A	)
				return scores






class _UpperCamelCase							(							a__   ):
			'''simple docstring'''







			def __init__(			self							: Tuple      ,      a							: Optional[Any]	)							->       Any:

				"""simple docstring"""

				if not isinstance(_A      ,      _A	) or not (temperature > 0):
					raise ValueError(F"`temperature` has to be a strictly positive float, but is {temperature}"	)

				SCREAMING_SNAKE_CASE   :      List[Any]				     =       temperature






			def __call__(			self							: Optional[int]      ,      a							: int      ,      a							: Union[str, Any]      ,      a							: Optional[Any]	)							->       Optional[int]:

				"""simple docstring"""

				SCREAMING_SNAKE_CASE   :      int				     =       scores / self.temperature
				return scores






class _UpperCamelCase							(							a__   ):
			'''simple docstring'''







			def __init__(			self							: Optional[int]      ,      a							: int      ,      a							: int = -float("Inf"	)      ,      a							: Optional[int] = 1	)							->       List[Any]:

				"""simple docstring"""

				if not isinstance(_A      ,      _A	) or (top_p < 0 or top_p > 1.0):
					raise ValueError(F"`top_p` has to be a float > 0 and < 1, but is {top_p}"	)
				if not isinstance(_A      ,      _A	) or (min_tokens_to_keep < 1):
					raise ValueError(F"`min_tokens_to_keep` has to be a positive integer, but is {min_tokens_to_keep}"	)

				SCREAMING_SNAKE_CASE   :      List[Any]				     =       top_p
				SCREAMING_SNAKE_CASE   :      str				     =       filter_value
				SCREAMING_SNAKE_CASE   :      Union[str, Any]				     =       min_tokens_to_keep






			def __call__(			self							: int      ,      a							: Optional[Any]      ,      a							: Optional[int]      ,      a							: Dict	)							->       Any:

				"""simple docstring"""

				SCREAMING_SNAKE_CASE   :      Union[str, Any]				     =       lax.top_k(_A      ,      scores.shape[-1]	)

				SCREAMING_SNAKE_CASE   :      Union[str, Any]				     =       jnp.full_like(_A      ,      self.filter_value	)
				SCREAMING_SNAKE_CASE   :      Optional[int]				     =       jax.nn.softmax(_A      ,      axis=-1	).cumsum(axis=-1	)
				SCREAMING_SNAKE_CASE   :      Optional[int]				     =       cumulative_probs < self.top_p

				# include the token that is higher than top_p as well
				SCREAMING_SNAKE_CASE   :      Optional[Any]				     =       jnp.roll(_A      ,      1	)
				score_mask |= score_mask.at[:, 0].set(_A	)

				# min tokens to keep
				SCREAMING_SNAKE_CASE   :      List[str]				     =       score_mask.at[:, : self.min_tokens_to_keep].set(_A	)

				SCREAMING_SNAKE_CASE   :      List[str]				     =       jnp.where(_A      ,      _A      ,      _A	)
				SCREAMING_SNAKE_CASE   :      Tuple				     =       jax.lax.sort_key_val(_A      ,      _A	)[-1]

				return next_scores






class _UpperCamelCase							(							a__   ):
			'''simple docstring'''







			def __init__(			self							: List[str]      ,      a							: Optional[Any]      ,      a							: Dict = -float("Inf"	)      ,      a							: Dict = 1	)							->       List[str]:

				"""simple docstring"""

				if not isinstance(_A      ,      _A	) or top_k <= 0:
					raise ValueError(F"`top_k` has to be a strictly positive integer, but is {top_k}"	)

				SCREAMING_SNAKE_CASE   :      Dict				     =       max(_A      ,      _A	)
				SCREAMING_SNAKE_CASE   :      Dict				     =       filter_value






			def __call__(			self							: List[str]      ,      a							: Dict      ,      a							: Tuple      ,      a							: int	)							->       Dict:

				"""simple docstring"""

				SCREAMING_SNAKE_CASE   :      Tuple				     =       scores.shape
				SCREAMING_SNAKE_CASE   :      Union[str, Any]				     =       jnp.full(batch_size * vocab_size      ,      self.filter_value	)

				SCREAMING_SNAKE_CASE   :      str				     =       min(self.top_k      ,      scores.shape[-1]	)  # Safety check
				SCREAMING_SNAKE_CASE   :      int				     =       lax.top_k(_A      ,      _A	)
				SCREAMING_SNAKE_CASE   :      Union[str, Any]				     =       jnp.broadcast_to((jnp.arange(_A	) * vocab_size)[:, None]      ,      (batch_size, topk)	).flatten()
				SCREAMING_SNAKE_CASE   :      Optional[int]				     =       topk_scores.flatten()
				SCREAMING_SNAKE_CASE   :      Optional[int]				     =       topk_indices.flatten() + shift

				SCREAMING_SNAKE_CASE   :      List[Any]				     =       next_scores_flat.at[topk_indices_flat].set(_A	)
				SCREAMING_SNAKE_CASE   :      Union[str, Any]				     =       next_scores_flat.reshape(_A      ,      _A	)
				return next_scores






class _UpperCamelCase							(							a__   ):
			'''simple docstring'''







			def __init__(			self							: int      ,      a							: Any	)							->       List[str]:

				"""simple docstring"""

				SCREAMING_SNAKE_CASE   :      Any				     =       bos_token_id






			def __call__(			self							: Tuple      ,      a							: List[Any]      ,      a							: Optional[Any]      ,      a							: Any	)							->       Optional[Any]:

				"""simple docstring"""

				SCREAMING_SNAKE_CASE   :      List[str]				     =       jnp.full(scores.shape      ,      -float("inf"	)	)

				SCREAMING_SNAKE_CASE   :      Any				     =       1 - jnp.bool_(cur_len - 1	)

				SCREAMING_SNAKE_CASE   :      Optional[Any]				     =       jnp.where(_A      ,      new_scores.at[:, self.bos_token_id].set(0	)      ,      _A	)

				return scores






class _UpperCamelCase							(							a__   ):
			'''simple docstring'''







			def __init__(			self							: List[Any]      ,      a							: Tuple      ,      a							: List[Any]	)							->       List[str]:

				"""simple docstring"""

				SCREAMING_SNAKE_CASE   :      Optional[Any]				     =       max_length
				SCREAMING_SNAKE_CASE   :      Dict				     =       eos_token_id






			def __call__(			self							: List[Any]      ,      a							: str      ,      a							: Union[str, Any]      ,      a							: List[str]	)							->       List[Any]:

				"""simple docstring"""

				SCREAMING_SNAKE_CASE   :      Any				     =       jnp.full(scores.shape      ,      -float("inf"	)	)

				SCREAMING_SNAKE_CASE   :      Optional[int]				     =       1 - jnp.bool_(cur_len - self.max_length + 1	)

				SCREAMING_SNAKE_CASE   :      Any				     =       jnp.where(_A      ,      new_scores.at[:, self.eos_token_id].set(0	)      ,      _A	)

				return scores






class _UpperCamelCase							(							a__   ):
			'''simple docstring'''







			def __init__(			self							: Tuple      ,      a							: List[str]      ,      a							: List[Any]	)							->       Tuple:

				"""simple docstring"""

				if not isinstance(_A      ,      _A	) or min_length < 0:
					raise ValueError(F"`min_length` has to be a positive integer, but is {min_length}"	)

				if not isinstance(_A      ,      _A	) or eos_token_id < 0:
					raise ValueError(F"`eos_token_id` has to be a positive integer, but is {eos_token_id}"	)

				SCREAMING_SNAKE_CASE   :      Union[str, Any]				     =       min_length
				SCREAMING_SNAKE_CASE   :      Dict				     =       eos_token_id






			def __call__(			self							: Optional[int]      ,      a							: Dict      ,      a							: Union[str, Any]      ,      a							: List[Any]	)							->       Any:

				"""simple docstring"""

				SCREAMING_SNAKE_CASE   :      Union[str, Any]				     =       1 - jnp.clip(cur_len - self.min_length      ,      0      ,      1	)

				SCREAMING_SNAKE_CASE   :      int				     =       jnp.where(_A      ,      scores.at[:, self.eos_token_id].set(-float("inf"	)	)      ,      _A	)

				return scores






class _UpperCamelCase							(							a__   ):
			'''simple docstring'''







			def __init__(			self							: Union[str, Any]      ,      a							: str      ,      a							: Tuple	)							->       Any:

				"""simple docstring"""

				SCREAMING_SNAKE_CASE   :      Any				     =       list(_A	)
				SCREAMING_SNAKE_CASE   :      str				     =       begin_index






			def __call__(			self							: Optional[Any]      ,      a							: Any      ,      a							: Any      ,      a							: Union[str, Any]	)							->       List[str]:

				"""simple docstring"""

				SCREAMING_SNAKE_CASE   :      Optional[Any]				     =       1 - jnp.bool_(cur_len - self.begin_index	)

				SCREAMING_SNAKE_CASE   :      List[Any]				     =       jnp.where(_A      ,      scores.at[:, self.begin_suppress_tokens].set(-float("inf"	)	)      ,      _A	)

				return scores






class _UpperCamelCase							(							a__   ):
			'''simple docstring'''







			def __init__(			self							: Optional[int]      ,      a							: Dict	)							->       str:

				"""simple docstring"""

				SCREAMING_SNAKE_CASE   :      List[Any]				     =       list(_A	)






			def __call__(			self							: Any      ,      a							: int      ,      a							: Tuple      ,      a							: Tuple	)							->       List[Any]:

				"""simple docstring"""

				SCREAMING_SNAKE_CASE   :      str				     =       scores.at[..., self.suppress_tokens].set(-float("inf"	)	)

				return scores






class _UpperCamelCase							(							a__   ):
			'''simple docstring'''







			def __init__(			self							: Tuple      ,      a							: int	)							->       str:

				"""simple docstring"""

				SCREAMING_SNAKE_CASE   :      Optional[Any]				     =       dict(_A	)
				# Converts the dictionary of format {index: token} containing the tokens to be forced to an array, where the
				# index of the array corresponds to the index of the token to be forced, for XLA compatibility.
				# Indexes without forced tokens will have a negative value.
				SCREAMING_SNAKE_CASE   :      int				     =       jnp.ones((max(force_token_map.keys()	) + 1)      ,      dtype=jnp.intaa	) * -1
				for index, token in force_token_map.items():
					if token is not None:
						SCREAMING_SNAKE_CASE   :      Optional[int]				     =       force_token_array.at[index].set(_A	)
				SCREAMING_SNAKE_CASE   :      str				     =       jnp.intaa(_A	)






			def __call__(			self							: str      ,      a							: Any      ,      a							: Optional[Any]      ,      a							: List[str]	)							->       Union[str, Any]:

				"""simple docstring"""

				def _force_token(a							: Optional[int]	):
					SCREAMING_SNAKE_CASE   :      List[str]				     =       scores.shape[0]
					SCREAMING_SNAKE_CASE   :      Tuple				     =       self.force_token_array[generation_idx]

					SCREAMING_SNAKE_CASE   :      int				     =       jnp.ones_like(_A      ,      dtype=scores.dtype	) * -float("inf"	)
					SCREAMING_SNAKE_CASE   :      List[Any]				     =       jnp.zeros((batch_size, 1)      ,      dtype=scores.dtype	)
					SCREAMING_SNAKE_CASE   :      Optional[Any]				     =       lax.dynamic_update_slice(_A      ,      _A      ,      (0, current_token)	)
					return new_scores

				SCREAMING_SNAKE_CASE   :      Optional[int]				     =       lax.cond(
				    cur_len >= self.force_token_array.shape[0]      ,      lambda: scores      ,      lambda: lax.cond(
				        self.force_token_array[cur_len] >= 0      ,      lambda: _force_token(_A	)      ,      lambda: scores      ,      )      ,      )
				return scores






class _UpperCamelCase							(							a__   ):
			'''simple docstring'''







			def __init__(			self							: Optional[int]      ,      a							: Optional[Any]      ,      a							: Union[str, Any]      ,      a							: Any	)							->       List[Any]:

				"""simple docstring"""

				SCREAMING_SNAKE_CASE   :      List[Any]				     =       generate_config.eos_token_id
				SCREAMING_SNAKE_CASE   :      Union[str, Any]				     =       generate_config.no_timestamps_token_id
				SCREAMING_SNAKE_CASE   :      List[str]				     =       generate_config.no_timestamps_token_id + 1

				SCREAMING_SNAKE_CASE   :      Optional[Any]				     =       decoder_input_length + 1

				if generate_config.is_multilingual:
					# room for language token and task token
					self.begin_index += 2
				if hasattr(_A      ,      "max_initial_timestamp_index"	):
					SCREAMING_SNAKE_CASE   :      Optional[Any]				     =       generate_config.max_initial_timestamp_index
				else:
					SCREAMING_SNAKE_CASE   :      Optional[int]				     =       model_config.vocab_size
				if self.max_initial_timestamp_index is None:
					SCREAMING_SNAKE_CASE   :      str				     =       model_config.vocab_size






			def __call__(			self							: Union[str, Any]      ,      a							: Any      ,      a							: int      ,      a							: int	)							->       int:

				"""simple docstring"""

				SCREAMING_SNAKE_CASE   :      str				     =       scores.at[:, self.no_timestamps_token_id].set(-float("inf"	)	)

				def handle_pairs(a							: Any      ,      a							: Tuple	):
					SCREAMING_SNAKE_CASE   :      str				     =       jnp.where((cur_len - self.begin_index) >= 1      ,      _A      ,      _A	)
					SCREAMING_SNAKE_CASE   :      Union[str, Any]				     =       jnp.where(
					    input_ids_k[cur_len - 1] >= self.timestamp_begin      ,      True and last_was_timestamp      ,      _A      ,      )

					SCREAMING_SNAKE_CASE   :      Any				     =       jnp.where((cur_len - self.begin_index) < 2      ,      _A      ,      _A	)
					SCREAMING_SNAKE_CASE   :      int				     =       jnp.where(
					    input_ids_k[cur_len - 2] >= self.timestamp_begin      ,      _A      ,      _A      ,      )

					return jnp.where(
					    _A      ,      jnp.where(
					        penultimate_was_timestamp > 0      ,      scores_k.at[self.timestamp_begin :].set(-float("inf"	)	)      ,      scores_k.at[: self.eos_token_id].set(-float("inf"	)	)      ,      )      ,      _A      ,      )

				SCREAMING_SNAKE_CASE   :      Dict				     =       jax.vmap(_A	)(_A      ,      _A	)

				SCREAMING_SNAKE_CASE   :      Tuple				     =       jnp.where(cur_len == self.begin_index      ,      _A      ,      _A	)
				SCREAMING_SNAKE_CASE   :      List[str]				     =       jnp.where(
				    self.max_initial_timestamp_index is not None      ,      True and apply_max_initial_timestamp      ,      _A      ,      )

				SCREAMING_SNAKE_CASE   :      Dict				     =       self.timestamp_begin + self.max_initial_timestamp_index

				SCREAMING_SNAKE_CASE   :      Dict				     =       jnp.where(
				    _A      ,      scores.at[:, last_allowed + 1 :].set(-float("inf"	)	)      ,      _A      ,      )

				# if sum of probability over timestamps is above any other token, sample timestamp
				SCREAMING_SNAKE_CASE   :      Dict				     =       jax.nn.log_softmax(_A      ,      axis=-1	)

				def handle_cumulative_probs(a							: Any      ,      a							: int	):
					SCREAMING_SNAKE_CASE   :      Tuple				     =       jax.nn.logsumexp(logprobs_k[self.timestamp_begin :]      ,      axis=-1	)
					SCREAMING_SNAKE_CASE   :      Tuple				     =       jnp.max(logprobs_k[: self.timestamp_begin]	)
					return jnp.where(
					    timestamp_logprob > max_text_token_logprob      ,      scores_k.at[: self.timestamp_begin].set(-float("inf"	)	)      ,      _A      ,      )

				SCREAMING_SNAKE_CASE   :      Any				     =       jax.vmap(_A	)(_A      ,      _A	)

				return scores


import os
import tempfile
import unittest

from transformers.models.marian.convert_marian_tatoeba_to_pytorch import DEFAULT_REPO, TatoebaConverter
from transformers.testing_utils import slow
from transformers.utils import cached_property






@unittest.skipUnless(os.path.exists(__A   )							,	'Tatoeba directory does not exist.'   )
class _UpperCamelCase							(							unittest.TestCase   ):
			'''simple docstring'''







			@cached_property
			def 							__UpperCamelCase (			self							: Any	)							->       str:

				"""simple docstring"""

				SCREAMING_SNAKE_CASE   :      Optional[Any]				     =       tempfile.mkdtemp()
				return TatoebaConverter(save_dir=a	)
			@slow
			def 							__UpperCamelCase (			self							: List[Any]	)							->       int:

				"""simple docstring"""

				self.resolver.convert_models(["heb-eng"]	)






			@slow
			def 							__UpperCamelCase (			self							: Optional[int]	)							->       Dict:

				"""simple docstring"""

				SCREAMING_SNAKE_CASE     ,SCREAMING_SNAKE_CASE   :      Optional[Any]				     =       self.resolver.write_model_card("opus-mt-he-en"      ,      dry_run=a	)
				assert mmeta["long_pair"] == "heb-eng"