
import logging
import os
from dataclasses import dataclass
from typing import List, Optional, Union

import tqdm
from filelock import FileLock

from transformers import (
    BartTokenizer,
    BartTokenizerFast,
    DataProcessor,
    PreTrainedTokenizer,
    RobertaTokenizer,
    RobertaTokenizerFast,
    XLMRobertaTokenizer,
    is_tf_available,
    is_torch_available,
)


UpperCamelCase_   	=	logging.getLogger(__name__)

@dataclass(frozen=__UpperCAmelCase					)
class       a		:
							lowercase_		:						str
							lowercase_		:						str
							lowercase_		:						Optional[str]   							=  None
							lowercase_		:						Optional[str]   							=  None
							lowercase_		:						Optional[str]   							=  None

@dataclass(frozen=__UpperCAmelCase					)
class       a		:
							lowercase_		:						List[int]
							lowercase_		:						Optional[List[int]]   							=  None
							lowercase_		:						Optional[List[int]]   							=  None
							lowercase_		:						Optional[Union[int, float]]   							=  None
							lowercase_		:						Optional[int]   							=  None


if is_torch_available():
					import torch
					from torch.utils.data import Dataset

					class       a		(   __UpperCAmelCase					):
												lowercase_		:						List[InputFeatures]







												def __init__(						self  :  List[str]      ,				snake_case__  :  str      ,				snake_case__  :  PreTrainedTokenizer      ,				snake_case__  :  str      ,				snake_case__  :  Optional[int] = None      ,				snake_case__  :  Optional[int]=False      ,				snake_case__  :  bool = False      ,				):
													"""simple docstring"""







													__lowerCAmelCase         =							hans_processors[task]()

													__lowerCAmelCase         =							os.path.join(
													    snake_case__      ,				"cached_{}_{}_{}_{}".format(
													        "dev" if evaluate else "train"      ,				tokenizer.__class__.__name__      ,				str(snake_case__ )      ,				snake_case__      ,				)      ,				)
													__lowerCAmelCase         =							processor.get_labels()
													if tokenizer.__class__ in (
													    RobertaTokenizer,
													    RobertaTokenizerFast,
													    XLMRobertaTokenizer,
													    BartTokenizer,
													    BartTokenizerFast,
													):
														# HACK(label indices are swapped in RoBERTa pretrained model)
														__lowerCAmelCase   ,						__lowerCAmelCase         =							label_list[2], label_list[1]
													__lowerCAmelCase         =							label_list

													# Make sure only the first process in distributed training processes the dataset,
													# and the others will use the cache.
													__lowerCAmelCase         =							cached_features_file + ".lock"
													with FileLock(snake_case__ ):
														if os.path.exists(snake_case__ ) and not overwrite_cache:
															logger.info(F"Loading features from cached file {cached_features_file}" )
															__lowerCAmelCase         =							torch.load(snake_case__ )
														else:
															logger.info(F"Creating features from dataset file at {data_dir}" )

															__lowerCAmelCase         =							(
															    processor.get_dev_examples(snake_case__ ) if evaluate else processor.get_train_examples(snake_case__ )
															)

															logger.info("Training examples: %s"      ,				len(snake_case__ ) )
															__lowerCAmelCase         =							hans_convert_examples_to_features(snake_case__      ,				snake_case__      ,				snake_case__      ,				snake_case__ )
															logger.info("Saving features into cached file %s"      ,				snake_case__ )
															torch.save(self.features      ,				snake_case__ )







												def __len__(						self  :  Optional[Any] ):
													"""simple docstring"""







													return len(self.features )







												def __getitem__(						self  :  List[str]      ,				snake_case__  :  Union[str, Any] ):
													"""simple docstring"""







													return self.features[i]







												def       UpperCAmelCase__					(						self  :  str ):
													"""simple docstring"""







													return self.label_list


if is_tf_available():
					import tensorflow as tf

					class       a		:
												lowercase_		:						List[InputFeatures]







												def __init__(						self  :  Tuple      ,				snake_case__  :  str      ,				snake_case__  :  PreTrainedTokenizer      ,				snake_case__  :  str      ,				snake_case__  :  Optional[int] = 128      ,				snake_case__  :  Optional[int]=False      ,				snake_case__  :  bool = False      ,				):
													"""simple docstring"""







													__lowerCAmelCase         =							hans_processors[task]()
													__lowerCAmelCase         =							processor.get_labels()
													if tokenizer.__class__ in (
													    RobertaTokenizer,
													    RobertaTokenizerFast,
													    XLMRobertaTokenizer,
													    BartTokenizer,
													    BartTokenizerFast,
													):
														# HACK(label indices are swapped in RoBERTa pretrained model)
														__lowerCAmelCase   ,						__lowerCAmelCase         =							label_list[2], label_list[1]
													__lowerCAmelCase         =							label_list

													__lowerCAmelCase         =							processor.get_dev_examples(snake_case__ ) if evaluate else processor.get_train_examples(snake_case__ )
													__lowerCAmelCase         =							hans_convert_examples_to_features(snake_case__      ,				snake_case__      ,				snake_case__      ,				snake_case__ )

													def gen():
														for ex_index, ex in tqdm.tqdm(enumerate(self.features )      ,				desc="convert examples to features" ):
															if ex_index % 10_000 == 0:
																logger.info("Writing example %d of %d" % (ex_index, len(snake_case__ )) )

															yield (
															    {
															        "example_id": 0,
															        "input_ids": ex.input_ids,
															        "attention_mask": ex.attention_mask,
															        "token_type_ids": ex.token_type_ids,
															    },
															    ex.label,
															)

													__lowerCAmelCase         =							tf.data.Dataset.from_generator(
													    snake_case__      ,				(
													        {
													            "example_id": tf.intaa,
													            "input_ids": tf.intaa,
													            "attention_mask": tf.intaa,
													            "token_type_ids": tf.intaa,
													        },
													        tf.intaa,
													    )      ,				(
													        {
													            "example_id": tf.TensorShape([] ),
													            "input_ids": tf.TensorShape([None, None] ),
													            "attention_mask": tf.TensorShape([None, None] ),
													            "token_type_ids": tf.TensorShape([None, None] ),
													        },
													        tf.TensorShape([] ),
													    )      ,				)







												def       UpperCAmelCase__					(						self  :  Optional[Any] ):
													"""simple docstring"""







													return self.dataset







												def __len__(						self  :  int ):
													"""simple docstring"""







													return len(self.features )







												def __getitem__(						self  :  Tuple      ,				snake_case__  :  List[Any] ):
													"""simple docstring"""







													return self.features[i]







												def       UpperCAmelCase__					(						self  :  Union[str, Any] ):
													"""simple docstring"""







													return self.label_list

class       a		(   __UpperCAmelCase					):







							def       UpperCAmelCase__					(						self  :  Dict      ,				snake_case__  :  List[str] ):
								"""simple docstring"""







								return self._create_examples(self._read_tsv(os.path.join(snake_case__      ,				"heuristics_train_set.txt" ) )      ,				"train" )







							def       UpperCAmelCase__					(						self  :  Dict      ,				snake_case__  :  Dict ):
								"""simple docstring"""







								return self._create_examples(self._read_tsv(os.path.join(snake_case__      ,				"heuristics_evaluation_set.txt" ) )      ,				"dev" )







							def       UpperCAmelCase__					(						self  :  Dict ):
								"""simple docstring"""







								return ["contradiction", "entailment", "neutral"]







							def       UpperCAmelCase__					(						self  :  Optional[int]      ,				snake_case__  :  List[str]      ,				snake_case__  :  List[Any] ):
								"""simple docstring"""







								__lowerCAmelCase         =							[]
								for i, line in enumerate(snake_case__ ):
									if i == 0:
										continue
									__lowerCAmelCase         =							"%s-%s" % (set_type, line[0])
									__lowerCAmelCase         =							line[5]
									__lowerCAmelCase         =							line[6]
									__lowerCAmelCase         =							line[7][2:] if line[7].startswith("ex" ) else line[7]
									__lowerCAmelCase         =							line[0]
									examples.append(InputExample(guid=snake_case__      ,				text_a=snake_case__      ,				text_b=snake_case__      ,				label=snake_case__      ,				pairID=snake_case__ ) )
								return examples






def 		_UpperCAmelCase   (	UpperCamelCase:     List[InputExample]			,							UpperCamelCase:     List[str]			,							UpperCamelCase:     int			,							UpperCamelCase:     PreTrainedTokenizer			,							):






	"""simple docstring"""


	__lowerCAmelCase         =							{label: i for i, label in enumerate(UpperCamelCase  )}

	__lowerCAmelCase         =							[]
	for ex_index, example in tqdm.tqdm(enumerate(UpperCamelCase  )			,							desc="convert examples to features"  ):
		if ex_index % 1_0_0_0_0 == 0:
			logger.info("Writing example %d" % (ex_index)  )

		__lowerCAmelCase         =							tokenizer(
		    example.text_a			,							example.text_b			,							add_special_tokens=UpperCamelCase			,							max_length=UpperCamelCase			,							padding="max_length"			,							truncation=UpperCamelCase			,							return_overflowing_tokens=UpperCamelCase			,							)

		__lowerCAmelCase         =							label_map[example.label] if example.label in label_map else 0

		__lowerCAmelCase         =							int(example.pairID  )

		features.append(InputFeatures(**UpperCamelCase			,							label=UpperCamelCase			,							pairID=UpperCamelCase  )  )

	for i, example in enumerate(examples[:5]  ):
		logger.info("*** Example ***"  )
		logger.info(F"guid: {example}"  )
		logger.info(F"features: {features[i]}"  )

	return features


UpperCamelCase_   	=	{
    "hans": 3,
}







UpperCamelCase_   	=	{
    "hans": HansProcessor,
}








import unittest

import numpy as np

from transformers import is_flax_available
from transformers.testing_utils import require_flax

from ..test_modeling_flax_common import ids_tensor


if is_flax_available():
					import jax
					import jax.numpy as jnp

					from transformers.generation import (
					    FlaxForcedBOSTokenLogitsProcessor,
					    FlaxForcedEOSTokenLogitsProcessor,
					    FlaxLogitsProcessorList,
					    FlaxMinLengthLogitsProcessor,
					    FlaxTemperatureLogitsWarper,
					    FlaxTopKLogitsWarper,
					    FlaxTopPLogitsWarper,
					)

@require_flax
class       a		(   unittest.TestCase					):







							def       UpperCAmelCase__					(						self  :  Tuple      ,				snake_case__  :  int      ,				snake_case__  :  int ):
								"""simple docstring"""







								__lowerCAmelCase         =							jnp.ones((batch_size, length) ) / length
								return scores







							def       UpperCAmelCase__					(						self  :  Dict ):
								"""simple docstring"""







								__lowerCAmelCase         =							None
								__lowerCAmelCase         =							20

								__lowerCAmelCase         =							self._get_uniform_logits(batch_size=2      ,				length=snake_case__ )

								# tweak scores to not be uniform anymore
								__lowerCAmelCase         =							scores.at[1, 5].set((1 / length) + 0.1 )  # peak, 1st batch
								__lowerCAmelCase         =							scores.at[1, 10].set((1 / length) - 0.4 )  # valley, 1st batch

								# compute softmax
								__lowerCAmelCase         =							jax.nn.softmax(snake_case__      ,				axis=-1 )

								__lowerCAmelCase         =							FlaxTemperatureLogitsWarper(temperature=0.5 )
								__lowerCAmelCase         =							FlaxTemperatureLogitsWarper(temperature=1.3 )

								__lowerCAmelCase         =							jax.nn.softmax(temp_dist_warper_sharper(snake_case__      ,				scores.copy()      ,				cur_len=snake_case__ )      ,				axis=-1 )
								__lowerCAmelCase         =							jax.nn.softmax(temp_dist_warper_smoother(snake_case__      ,				scores.copy()      ,				cur_len=snake_case__ )      ,				axis=-1 )

								# uniform distribution stays uniform
								self.assertTrue(jnp.allclose(probs[0, :]      ,				warped_prob_sharp[0, :]      ,				atol=1E-3 ) )
								self.assertTrue(jnp.allclose(probs[0, :]      ,				warped_prob_smooth[0, :]      ,				atol=1E-3 ) )

								# sharp peaks get higher, valleys get lower
								self.assertLess(probs[1, :].max()      ,				warped_prob_sharp[1, :].max() )
								self.assertGreater(probs[1, :].min()      ,				warped_prob_sharp[1, :].min() )

								# smooth peaks get lower, valleys get higher
								self.assertGreater(probs[1, :].max()      ,				warped_prob_smooth[1, :].max() )
								self.assertLess(probs[1, :].min()      ,				warped_prob_smooth[1, :].min() )







							def       UpperCAmelCase__					(						self  :  str ):
								"""simple docstring"""







								__lowerCAmelCase         =							None
								__lowerCAmelCase         =							10
								__lowerCAmelCase         =							2

								# create ramp distribution
								__lowerCAmelCase         =							np.broadcast_to(np.arange(snake_case__ )[None, :]      ,				(batch_size, vocab_size) ).copy()
								__lowerCAmelCase         =							ramp_logits[1:, : vocab_size // 2] + vocab_size

								__lowerCAmelCase         =							FlaxTopKLogitsWarper(3 )

								__lowerCAmelCase         =							top_k_warp(snake_case__      ,				snake_case__      ,				cur_len=snake_case__ )

								# check that correct tokens are filtered
								self.assertListEqual(jnp.isinf(scores[0] ).tolist()      ,				7 * [True] + 3 * [False] )
								self.assertListEqual(jnp.isinf(scores[1] ).tolist()      ,				2 * [True] + 3 * [False] + 5 * [True] )

								# check special case
								__lowerCAmelCase         =							5
								__lowerCAmelCase         =							FlaxTopKLogitsWarper(top_k=1      ,				filter_value=0.0      ,				min_tokens_to_keep=3 )

								__lowerCAmelCase         =							np.broadcast_to(np.arange(snake_case__ )[None, :]      ,				(batch_size, length) ).copy()
								__lowerCAmelCase         =							top_k_warp_safety_check(snake_case__      ,				snake_case__      ,				cur_len=snake_case__ )

								# min_tokens overwrites k: 3 tokens are kept => 2 tokens are nullified
								self.assertListEqual((scores == 0.0).sum(axis=-1 ).tolist()      ,				[2, 2] )







							def       UpperCAmelCase__					(						self  :  int ):
								"""simple docstring"""







								__lowerCAmelCase         =							None
								__lowerCAmelCase         =							10
								__lowerCAmelCase         =							2

								# create distribution and take log (inverse to Softmax as taken in TopPLogitsWarper)
								__lowerCAmelCase         =							np.log(np.array([[0.3, 0.1, 0.1, 0.5], [0.1_5, 0.3, 0.3, 0.2_5]] ) )

								__lowerCAmelCase         =							FlaxTopPLogitsWarper(0.8 )
								__lowerCAmelCase         =							np.exp(top_p_warp(snake_case__      ,				snake_case__      ,				cur_len=snake_case__ ) )

								# dist should be filtered to keep min num values so that sum is >= top_p
								# exp (-inf) => 0
								__lowerCAmelCase         =							np.array([[0.3, 0.0, 0.0, 0.5], [0.0, 0.3, 0.3, 0.2_5]] )
								self.assertTrue(np.allclose(snake_case__      ,				snake_case__      ,				atol=1E-3 ) )

								# check edge cases with negative and extreme logits
								__lowerCAmelCase         =							np.broadcast_to(np.arange(snake_case__ )[None, :]      ,				(batch_size, vocab_size) ).copy() - (
								    vocab_size // 2
								)

								# make ramp_logits more extreme
								__lowerCAmelCase         =							ramp_logits[1] * 1_0_0.0

								# make sure at least 2 tokens are kept
								__lowerCAmelCase         =							FlaxTopPLogitsWarper(0.9      ,				min_tokens_to_keep=2      ,				filter_value=0.0 )
								__lowerCAmelCase         =							top_p_warp(snake_case__      ,				snake_case__      ,				cur_len=snake_case__ )

								# first batch should keep three tokens, second batch would keep only 1, but due to `min_tokens_to_keep=2` keeps 2.
								self.assertListEqual((filtered_dist != 0.0).sum(axis=-1 ).tolist()      ,				[3, 2] )







							def       UpperCAmelCase__					(						self  :  Dict ):
								"""simple docstring"""







								__lowerCAmelCase         =							20
								__lowerCAmelCase         =							4
								__lowerCAmelCase         =							0

								__lowerCAmelCase         =							FlaxMinLengthLogitsProcessor(min_length=10      ,				eos_token_id=snake_case__ )

								# check that min length is applied at length 5
								__lowerCAmelCase         =							ids_tensor((batch_size, 20)      ,				vocab_size=20 )
								__lowerCAmelCase         =							5
								__lowerCAmelCase         =							self._get_uniform_logits(snake_case__      ,				snake_case__ )
								__lowerCAmelCase         =							min_dist_processor(snake_case__      ,				snake_case__      ,				cur_len=snake_case__ )
								self.assertListEqual(scores_before_min_length[:, eos_token_id].tolist()      ,				4 * [-float("inf" )] )

								# check that min length is not applied anymore at length 15
								__lowerCAmelCase         =							self._get_uniform_logits(snake_case__      ,				snake_case__ )
								__lowerCAmelCase         =							15
								__lowerCAmelCase         =							min_dist_processor(snake_case__      ,				snake_case__      ,				cur_len=snake_case__ )
								self.assertFalse(jnp.isinf(snake_case__ ).any() )







							def       UpperCAmelCase__					(						self  :  int ):
								"""simple docstring"""







								__lowerCAmelCase         =							20
								__lowerCAmelCase         =							4
								__lowerCAmelCase         =							0

								__lowerCAmelCase         =							FlaxForcedBOSTokenLogitsProcessor(bos_token_id=snake_case__ )

								# check that all scores are -inf except the bos_token_id score
								__lowerCAmelCase         =							ids_tensor((batch_size, 1)      ,				vocab_size=20 )
								__lowerCAmelCase         =							1
								__lowerCAmelCase         =							self._get_uniform_logits(snake_case__      ,				snake_case__ )
								__lowerCAmelCase         =							logits_processor(snake_case__      ,				snake_case__      ,				cur_len=snake_case__ )
								self.assertTrue(jnp.isneginf(scores[:, bos_token_id + 1 :] ).all() )
								self.assertListEqual(scores[:, bos_token_id].tolist()      ,				4 * [0] )  # score for bos_token_id shold be zero

								# check that bos_token_id is not forced if current length is greater than 1
								__lowerCAmelCase         =							3
								__lowerCAmelCase         =							self._get_uniform_logits(snake_case__      ,				snake_case__ )
								__lowerCAmelCase         =							logits_processor(snake_case__      ,				snake_case__      ,				cur_len=snake_case__ )
								self.assertFalse(jnp.isinf(snake_case__ ).any() )







							def       UpperCAmelCase__					(						self  :  str ):
								"""simple docstring"""







								__lowerCAmelCase         =							20
								__lowerCAmelCase         =							4
								__lowerCAmelCase         =							0
								__lowerCAmelCase         =							5

								__lowerCAmelCase         =							FlaxForcedEOSTokenLogitsProcessor(max_length=snake_case__      ,				eos_token_id=snake_case__ )

								# check that all scores are -inf except the eos_token_id when max_length is reached
								__lowerCAmelCase         =							ids_tensor((batch_size, 4)      ,				vocab_size=20 )
								__lowerCAmelCase         =							4
								__lowerCAmelCase         =							self._get_uniform_logits(snake_case__      ,				snake_case__ )
								__lowerCAmelCase         =							logits_processor(snake_case__      ,				snake_case__      ,				cur_len=snake_case__ )
								self.assertTrue(jnp.isneginf(scores[:, eos_token_id + 1 :] ).all() )
								self.assertListEqual(scores[:, eos_token_id].tolist()      ,				4 * [0] )  # score for eos_token_id should be zero

								# check that eos_token_id is not forced if max_length is not reached
								__lowerCAmelCase         =							3
								__lowerCAmelCase         =							self._get_uniform_logits(snake_case__      ,				snake_case__ )
								__lowerCAmelCase         =							logits_processor(snake_case__      ,				snake_case__      ,				cur_len=snake_case__ )
								self.assertFalse(jnp.isinf(snake_case__ ).any() )







							def       UpperCAmelCase__					(						self  :  Dict ):
								"""simple docstring"""







								__lowerCAmelCase         =							4
								__lowerCAmelCase         =							10
								__lowerCAmelCase         =							15
								__lowerCAmelCase         =							2
								__lowerCAmelCase         =							1
								__lowerCAmelCase         =							15

								# dummy input_ids and scores
								__lowerCAmelCase         =							ids_tensor((batch_size, sequence_length)      ,				snake_case__ )
								__lowerCAmelCase         =							input_ids.copy()

								__lowerCAmelCase         =							self._get_uniform_logits(snake_case__      ,				snake_case__ )
								__lowerCAmelCase         =							scores.copy()

								# instantiate all dist processors
								__lowerCAmelCase         =							FlaxTemperatureLogitsWarper(temperature=0.5 )
								__lowerCAmelCase         =							FlaxTopKLogitsWarper(3 )
								__lowerCAmelCase         =							FlaxTopPLogitsWarper(0.8 )

								# instantiate all logits processors
								__lowerCAmelCase         =							FlaxMinLengthLogitsProcessor(min_length=10      ,				eos_token_id=snake_case__ )
								__lowerCAmelCase         =							FlaxForcedBOSTokenLogitsProcessor(bos_token_id=snake_case__ )
								__lowerCAmelCase         =							FlaxForcedEOSTokenLogitsProcessor(max_length=snake_case__      ,				eos_token_id=snake_case__ )

								__lowerCAmelCase         =							10

								# no processor list
								__lowerCAmelCase         =							temp_dist_warp(snake_case__      ,				snake_case__      ,				cur_len=snake_case__ )
								__lowerCAmelCase         =							top_k_warp(snake_case__      ,				snake_case__      ,				cur_len=snake_case__ )
								__lowerCAmelCase         =							top_p_warp(snake_case__      ,				snake_case__      ,				cur_len=snake_case__ )
								__lowerCAmelCase         =							min_dist_proc(snake_case__      ,				snake_case__      ,				cur_len=snake_case__ )
								__lowerCAmelCase         =							bos_dist_proc(snake_case__      ,				snake_case__      ,				cur_len=snake_case__ )
								__lowerCAmelCase         =							eos_dist_proc(snake_case__      ,				snake_case__      ,				cur_len=snake_case__ )

								# with processor list
								__lowerCAmelCase         =							FlaxLogitsProcessorList(
								    [temp_dist_warp, top_k_warp, top_p_warp, min_dist_proc, bos_dist_proc, eos_dist_proc] )
								__lowerCAmelCase         =							processor(snake_case__      ,				snake_case__      ,				cur_len=snake_case__ )

								# scores should be equal
								self.assertTrue(jnp.allclose(snake_case__      ,				snake_case__      ,				atol=1E-3 ) )

								# input_ids should never be changed
								self.assertListEqual(input_ids.tolist()      ,				input_ids_comp.tolist() )







							def       UpperCAmelCase__					(						self  :  Dict ):
								"""simple docstring"""







								__lowerCAmelCase         =							4
								__lowerCAmelCase         =							10
								__lowerCAmelCase         =							15
								__lowerCAmelCase         =							2
								__lowerCAmelCase         =							1
								__lowerCAmelCase         =							15

								# dummy input_ids and scores
								__lowerCAmelCase         =							ids_tensor((batch_size, sequence_length)      ,				snake_case__ )
								__lowerCAmelCase         =							input_ids.copy()

								__lowerCAmelCase         =							self._get_uniform_logits(snake_case__      ,				snake_case__ )
								__lowerCAmelCase         =							scores.copy()

								# instantiate all dist processors
								__lowerCAmelCase         =							FlaxTemperatureLogitsWarper(temperature=0.5 )
								__lowerCAmelCase         =							FlaxTopKLogitsWarper(3 )
								__lowerCAmelCase         =							FlaxTopPLogitsWarper(0.8 )

								# instantiate all logits processors
								__lowerCAmelCase         =							FlaxMinLengthLogitsProcessor(min_length=10      ,				eos_token_id=snake_case__ )
								__lowerCAmelCase         =							FlaxForcedBOSTokenLogitsProcessor(bos_token_id=snake_case__ )
								__lowerCAmelCase         =							FlaxForcedEOSTokenLogitsProcessor(max_length=snake_case__      ,				eos_token_id=snake_case__ )

								__lowerCAmelCase         =							10

								# no processor list
								def run_no_processor_list(snake_case__  :  int      ,				snake_case__  :  Any      ,				snake_case__  :  int ):
									__lowerCAmelCase         =							temp_dist_warp(snake_case__      ,				snake_case__      ,				cur_len=snake_case__ )
									__lowerCAmelCase         =							top_k_warp(snake_case__      ,				snake_case__      ,				cur_len=snake_case__ )
									__lowerCAmelCase         =							top_p_warp(snake_case__      ,				snake_case__      ,				cur_len=snake_case__ )
									__lowerCAmelCase         =							min_dist_proc(snake_case__      ,				snake_case__      ,				cur_len=snake_case__ )
									__lowerCAmelCase         =							bos_dist_proc(snake_case__      ,				snake_case__      ,				cur_len=snake_case__ )
									__lowerCAmelCase         =							eos_dist_proc(snake_case__      ,				snake_case__      ,				cur_len=snake_case__ )
									return scores

								# with processor list
								def run_processor_list(snake_case__  :  Optional[Any]      ,				snake_case__  :  Optional[int]      ,				snake_case__  :  str ):
									__lowerCAmelCase         =							FlaxLogitsProcessorList(
									    [temp_dist_warp, top_k_warp, top_p_warp, min_dist_proc, bos_dist_proc, eos_dist_proc] )
									__lowerCAmelCase         =							processor(snake_case__      ,				snake_case__      ,				cur_len=snake_case__ )
									return scores

								__lowerCAmelCase         =							jax.jit(snake_case__ )
								__lowerCAmelCase         =							jax.jit(snake_case__ )

								__lowerCAmelCase         =							jitted_run_no_processor_list(snake_case__      ,				snake_case__      ,				snake_case__ )
								__lowerCAmelCase         =							jitted_run_processor_list(snake_case__      ,				snake_case__      ,				snake_case__ )

								# scores should be equal
								self.assertTrue(jnp.allclose(snake_case__      ,				snake_case__      ,				atol=1E-3 ) )

								# input_ids should never be changed
								self.assertListEqual(input_ids.tolist()      ,				input_ids_comp.tolist() )






