




import collections
import json
import os
import re
from typing import TYPE_CHECKING, List, Optional, Tuple

import numpy as np

from ...tokenization_utils_fast import PreTrainedTokenizer
from ...utils import logging


if TYPE_CHECKING:
    from transformers.pipelines.conversational import Conversation


_lowerCamelCase			:	Tuple							 =     logging.get_logger(__name__)

_lowerCamelCase			:	str							 =     {"""vocab_file""": """vocab.txt""", """emoji_file""": """emoji.json"""}

_lowerCamelCase			:	Union[str, Any]							 =     {
    """vocab_file""": {
        """abeja/gpt-neox-japanese-2.7b""": """https://huggingface.co/abeja/gpt-neox-japanese-2.7b/resolve/main/vocab.txt""",
    },
    """emoji_file""": {
        """abeja/gpt-neox-japanese-2.7b""": """https://huggingface.co/abeja/gpt-neox-japanese-2.7b/resolve/main/emoji.json""",
    },
}

_lowerCamelCase			:	Tuple							 =     {
    """abeja/gpt-neox-japanese-2.7b""": 2048,
}







def SCREAMING_SNAKE_CASE       ( lowercase_				,       lowercase_		)     -> Optional[int]:



       """simple docstring"""

       with open(lowercase_				,       '''r'''				,       encoding='''utf-8'''		) as f:
              A__   				=					json.loads(f.read()		)

       A__   				=					collections.OrderedDict()
       A__   				=					collections.OrderedDict()
       A__   				=					collections.OrderedDict()
       with open(lowercase_				,       '''r'''				,       encoding='''utf-8'''		) as f:
              A__   				=					f.readlines()
       A__   				=					[[t.rstrip('''\n'''		)] if (t == ''',''' or ''',''' not in t) else t.rstrip('''\n'''		).split(''','''		) for t in token]
       for idx, b in enumerate(lowercase_		):
              A__   				=					b
              A__   				=					idx
              for wd in b:
                     A__   				=					idx

       return vocab, raw_vocab, ids_to_tokens, emoji


class        UpperCamelCase_					(				UpperCAmelCase__  ):

    '''simple docstring'''






    UpperCAmelCase__													=   VOCAB_FILES_NAMES
    UpperCAmelCase__													=   PRETRAINED_VOCAB_FILES_MAP
    UpperCAmelCase__													=   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    UpperCAmelCase__													=   ['''input_ids''', '''attention_mask''']


    def __init__(    self   :			Dict      ,				UpperCAmelCase__   :			Tuple      ,				UpperCAmelCase__   :			Optional[Any]      ,				UpperCAmelCase__   :			Optional[Any]="<|endoftext|>"      ,				UpperCAmelCase__   :			Optional[int]="<|endoftext|>"      ,				UpperCAmelCase__   :			Any="<|startoftext|>"      ,				UpperCAmelCase__   :			List[Any]="<|endoftext|>"      ,				UpperCAmelCase__   :			Dict=False      ,				**UpperCAmelCase__   :			Tuple      ,				)		->str:





           '''simple docstring'''

           super().__init__(
               unk_token=UpperCAmelCase__      ,				pad_token=UpperCAmelCase__      ,				bos_token=UpperCAmelCase__      ,				eos_token=UpperCAmelCase__      ,				do_clean_text=UpperCAmelCase__      ,				**UpperCAmelCase__      ,				)
           if not os.path.isfile(UpperCAmelCase__):
                  raise ValueError(
                      f"""Can't find a vocabulary file at path '{vocab_file}'. To load the vocabulary from a Google pretrained"""
                      ''' model use `tokenizer = GPTNeoXJapaneseokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`''')
           if not os.path.isfile(UpperCAmelCase__):
                  raise ValueError(
                      f"""Can't find a emoji file at path '{emoji_file}'. To load the emoji information from a Google"""
                      ''' pretrained model use `tokenizer = GPTNeoXJapaneseokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`''')
           A__   				=					do_clean_text
           A__				,							A__				,							A__				,							A__   				=					load_vocab_and_emoji(UpperCAmelCase__      ,				UpperCAmelCase__)
           A__   				=					SubWordJapaneseTokenizer(
               vocab=self.vocab      ,				ids_to_tokens=self.ids_to_tokens      ,				emoji=self.emoji)


    @property
    def  SCREAMING_SNAKE_CASE							(    self   :			Optional[Any])		->str:





           '''simple docstring'''

           return len(self.raw_vocab)


    def  SCREAMING_SNAKE_CASE							(    self   :			List[str])		->int:





           '''simple docstring'''

           return dict(self.raw_vocab      ,				**self.added_tokens_encoder)


    def  SCREAMING_SNAKE_CASE							(    self   :			Optional[int]      ,				UpperCAmelCase__   :			str)		->Union[str, Any]:





           '''simple docstring'''

           return self.subword_tokenizer.tokenize(UpperCAmelCase__      ,				clean=self.do_clean_text)


    def  SCREAMING_SNAKE_CASE							(    self   :			Dict      ,				UpperCAmelCase__   :			Union[str, Any])		->Optional[Any]:





           '''simple docstring'''

           return self.vocab.get(UpperCAmelCase__      ,				self.vocab.get(self.unk_token))


    def  SCREAMING_SNAKE_CASE							(    self   :			Dict      ,				UpperCAmelCase__   :			Optional[Any])		->str:





           '''simple docstring'''

           return self.subword_tokenizer.convert_id_to_token(UpperCAmelCase__)


    def  SCREAMING_SNAKE_CASE							(    self   :			str      ,				UpperCAmelCase__   :			Optional[Any])		->str:





           '''simple docstring'''

           A__   				=					''''''.join(UpperCAmelCase__).strip()
           return out_string


    def  SCREAMING_SNAKE_CASE							(    self   :			List[str]      ,				UpperCAmelCase__   :			"Conversation")		->List[int]:





           '''simple docstring'''

           A__   				=					[]
           for is_user, text in conversation.iter_texts():
                  input_ids.extend(self.encode(UpperCAmelCase__      ,				add_special_tokens=UpperCAmelCase__) + [self.eos_token_id])

           if len(UpperCAmelCase__) > self.model_max_length:
                  A__   				=					input_ids[-self.model_max_length :]
           return input_ids





    def  SCREAMING_SNAKE_CASE							(    self   :			str      ,				UpperCAmelCase__   :			str      ,				UpperCAmelCase__   :			Optional[str] = None)		->Tuple[str]:





           '''simple docstring'''

           A__   				=					0
           if os.path.isdir(UpperCAmelCase__):
                  A__   				=					os.path.join(
                      UpperCAmelCase__      ,				(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file'''])
                  A__   				=					os.path.join(
                      UpperCAmelCase__      ,				(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''emoji_file'''])
           else:
                  A__   				=					(
                      (filename_prefix + '''-''' if filename_prefix else '''''') + save_directory + VOCAB_FILES_NAMES['''vocab_file''']
                  )
                  A__   				=					(
                      (filename_prefix + '''-''' if filename_prefix else '''''') + save_directory + VOCAB_FILES_NAMES['''emoji_file''']
                  )
           with open(UpperCAmelCase__      ,				'''w'''      ,				encoding='''utf-8''') as writer:
                  for token_index, token in self.ids_to_tokens.items():
                         if index != token_index:
                                logger.warning(
                                    f"""Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive."""
                                    ''' Please check that the vocabulary is not corrupted!''')
                                A__   				=					token_index
                         writer.write(''','''.join(UpperCAmelCase__) + '''\n''')
                         index += 1
           with open(UpperCAmelCase__      ,				'''w'''      ,				encoding='''utf-8''') as writer:
                  json.dump(self.emoji      ,				UpperCAmelCase__)
           return vocab_file, emoji_file


class        UpperCamelCase_					(				UpperCAmelCase__  ):

    '''simple docstring'''






    def __init__(    self   :			Optional[int]      ,				UpperCAmelCase__   :			Union[str, Any]      ,				UpperCAmelCase__   :			List[str]      ,				UpperCAmelCase__   :			Optional[Any])		->List[str]:





           '''simple docstring'''

           A__   				=					vocab  # same as swe
           A__   				=					ids_to_tokens  # same as bpe
           A__   				=					emoji
           A__   				=					np.max([len(UpperCAmelCase__) for w in self.vocab.keys()])
           A__   				=					re.compile(R'''(https?|ftp)(:\/\/[-_\.!~*\'()a-zA-Z0-9;\/?:\@&=\+$,%#]+)''')
           A__   				=					re.compile(R'''[A-Za-z0-9\._+]*@[\-_0-9A-Za-z]+(\.[A-Za-z]+)*''')
           A__   				=					re.compile(R'''[\(]{0,1}[0-9]{2,4}[\)\-\(]{0,1}[0-9]{2,4}[\)\-]{0,1}[0-9]{3,4}''')
           A__   				=					re.compile(
               R'''([12]\d{3}[/\-年])*(0?[1-9]|1[0-2])[/\-月]((0?[1-9]|[12][0-9]|3[01])日?)*(\d{1,2}|:|\d{1,2}時|\d{1,2}分|\(日\)|\(月\)|\(火\)|\(水\)|\(木\)|\(金\)|\(土\)|㈰|㈪|㈫|㈬|㈭|㈮|㈯)*''')
           A__   				=					re.compile(
               R'''(明治|大正|昭和|平成|令和|㍾|㍽|㍼|㍻|\u32ff)\d{1,2}年(0?[1-9]|1[0-2])月(0?[1-9]|[12][0-9]|3[01])日(\d{1,2}|:|\d{1,2}時|\d{1,2}分|\(日\)|\(月\)|\(火\)|\(水\)|\(木\)|\(金\)|\(土\)|㈰|㈪|㈫|㈬|㈭|㈮|㈯)*''')
           A__   				=					re.compile(
               R'''((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*億)*((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*万)*((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*千)*(0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*(千円|万円|千万円|円|千ドル|万ドル|千万ドル|ドル|千ユーロ|万ユーロ|千万ユーロ|ユーロ)+(\(税込\)|\(税抜\)|\+tax)*''')
           A__   				=					'''─━│┃┄┅┆┇┈┉┊┋┌┍┎┏┐┑┒┓└┕┖┗┘┙┚┛├┝┞┟┠┡┢┣┤┥┦┧┨┩┪┫┬┭┮┯┰┱┲┳┴┵┶┷┸┹┺┻┼┽┾┿╀╁╂╃╄╅╆╇╈╉╊╋╌╍╎╏═║╒╓╔╕╖╗╘╙╚╛╜╝╞╟╠╡╢╣╤╥╦╧╨╩╪╫╬╭╮╯╰╱╲╳╴╵╶╷╸╹╺╻╼╽╾╿'''
           A__   				=					'''▀▁▂▃▄▅▆▇█▉▊▋▌▍▎▏▐░▒▓▔▕▖▗▘▙▚▛▜▝▞▟'''
           A__   				=					str.maketrans({k: '''<BLOCK>''' for k in keisen + blocks})


    def __len__(    self   :			Union[str, Any])		->List[Any]:





           '''simple docstring'''

           return len(self.ids_to_tokens)


    def  SCREAMING_SNAKE_CASE							(    self   :			str      ,				UpperCAmelCase__   :			List[Any])		->Dict:





           '''simple docstring'''

           A__   				=					self.content_repattera.sub('''<URL>'''      ,				UpperCAmelCase__)
           A__   				=					self.content_repattera.sub('''<EMAIL>'''      ,				UpperCAmelCase__)
           A__   				=					self.content_repattera.sub('''<TEL>'''      ,				UpperCAmelCase__)
           A__   				=					self.content_repattera.sub('''<DATE>'''      ,				UpperCAmelCase__)
           A__   				=					self.content_repattera.sub('''<DATE>'''      ,				UpperCAmelCase__)
           A__   				=					self.content_repattera.sub('''<PRICE>'''      ,				UpperCAmelCase__)
           A__   				=					content.translate(self.content_transa)
           while "<BLOCK><BLOCK>" in content:
                  A__   				=					content.replace('''<BLOCK><BLOCK>'''      ,				'''<BLOCK>''')
           return content


    def  SCREAMING_SNAKE_CASE							(    self   :			Optional[int]      ,				UpperCAmelCase__   :			Optional[int]      ,				UpperCAmelCase__   :			str=False)		->List[Any]:





           '''simple docstring'''

           A__   				=					text.replace(''' '''      ,				'''<SP>''')
           A__   				=					text.replace('''　'''      ,				'''<SP>''')
           A__   				=					text.replace('''\r\n'''      ,				'''<BR>''')
           A__   				=					text.replace('''\n'''      ,				'''<BR>''')
           A__   				=					text.replace('''\r'''      ,				'''<BR>''')
           A__   				=					text.replace('''\t'''      ,				'''<TAB>''')
           A__   				=					text.replace('''—'''      ,				'''ー''')
           A__   				=					text.replace('''−'''      ,				'''ー''')
           for k, v in self.emoji["emoji"].items():
                  if k in text:
                         A__   				=					text.replace(UpperCAmelCase__      ,				UpperCAmelCase__)
           if clean:
                  A__   				=					self.clean_text(UpperCAmelCase__)

           def check_simbol(UpperCAmelCase__   :			str):
                  A__   				=					x.encode()
                  if len(UpperCAmelCase__) == 1 and len(UpperCAmelCase__) == 2:
                         A__   				=					(int(e[0]) << 8) + int(e[1])
                         if (
                             (c >= 0xc2_a1 and c <= 0xc2_bf)
                             or (c >= 0xc7_80 and c <= 0xc7_83)
                             or (c >= 0xca_b9 and c <= 0xcb_bf)
                             or (c >= 0xcc_80 and c <= 0xcd_a2)
                         ):
                                return True
                  return False

           def checkuae(UpperCAmelCase__   :			Union[str, Any]):
                  A__   				=					x.encode()
                  if len(UpperCAmelCase__) == 1 and len(UpperCAmelCase__) == 3:
                         A__   				=					(int(e[0]) << 16) + (int(e[1]) << 8) + int(e[2])
                         if c >= 0xe2_80_80 and c <= 0xe2_b0_7f:
                                return True
                  return False

           A__   				=					0
           A__   				=					[]
           while pos < len(UpperCAmelCase__):
                  A__   				=					min(len(UpperCAmelCase__)      ,				pos + self.maxlen + 1) if text[pos] == '''<''' else pos + 3
                  A__   				=					[]  # (token_id, token, pos)
                  for e in range(UpperCAmelCase__      ,				UpperCAmelCase__      ,				-1):
                         A__   				=					text[pos:e]
                         if wd in self.vocab:
                                if wd[0] == "<" and len(UpperCAmelCase__) > 2:
                                       A__   				=					[(self.vocab[wd], wd, e)]
                                       break
                                else:
                                       candidates.append((self.vocab[wd], wd, e))
                  if len(UpperCAmelCase__) > 0:
                         # the smallest token_id is adopted
                         A__				,							A__				,							A__   				=					sorted(UpperCAmelCase__      ,				key=lambda UpperCAmelCase__: x[0])[0]
                         result.append(UpperCAmelCase__)
                         A__   				=					e
                  else:
                         A__   				=					pos + 1
                         A__   				=					text[pos:end]
                         if check_simbol(UpperCAmelCase__):
                                result.append('''<KIGOU>''')
                         elif checkuae(UpperCAmelCase__):
                                result.append('''<U2000U2BFF>''')
                         else:
                                for i in wd.encode('''utf-8'''):
                                       result.append('''<|byte%d|>''' % i)
                         A__   				=					end
           return result





    def  SCREAMING_SNAKE_CASE							(    self   :			int      ,				UpperCAmelCase__   :			str      ,				UpperCAmelCase__   :			Dict="\n")		->Dict:





           '''simple docstring'''

           A__   				=					[]
           A__   				=					[]
           A__   				=					self.ids_to_tokens[index][0]
           if word[:6] == "<|byte" and word[-2:] == "|>":
                  byte_tokens.append(int(word[6:-2]))
           else:
                  if len(UpperCAmelCase__) > 0:
                         words.append(bytearray(UpperCAmelCase__).decode('''utf-8'''      ,				errors='''replace'''))
                         A__   				=					[]
                  if word[:7] == "<|emoji" and word[-2:] == "|>":
                         words.append(self.emoji['''emoji_inv'''][word])
                  elif word == "<SP>":
                         words.append(''' ''')
                  elif word == "<BR>":
                         words.append(UpperCAmelCase__)
                  elif word == "<TAB>":
                         words.append('''\t''')
                  elif word == "<BLOCK>":
                         words.append('''▀''')
                  elif word == "<KIGOU>":
                         words.append('''ǀ''')
                  elif word == "<U2000U2BFF>":
                         words.append('''‖''')
                  else:
                         words.append(UpperCAmelCase__)
           if len(UpperCAmelCase__) > 0:
                  words.append(bytearray(UpperCAmelCase__).decode('''utf-8'''      ,				errors='''replace'''))
           A__   				=					''''''.join(UpperCAmelCase__)
           return text








import unittest

import numpy as np

from transformers import is_flax_available
from transformers.testing_utils import require_flax

from ..test_modeling_flax_common import ids_tensor


if is_flax_available():
  import jax
  import jax.numpy as jnp

  from transformers.generation import (
      FlaxForcedBOSTokenLogitsProcessor,
      FlaxForcedEOSTokenLogitsProcessor,
      FlaxLogitsProcessorList,
      FlaxMinLengthLogitsProcessor,
      FlaxTemperatureLogitsWarper,
      FlaxTopKLogitsWarper,
      FlaxTopPLogitsWarper,
  )


@require_flax
class        UpperCamelCase_					(				unittest.TestCase  ):

    '''simple docstring'''






    def  SCREAMING_SNAKE_CASE							(    self   :			int      ,				UpperCAmelCase__   :			int      ,				UpperCAmelCase__   :			int)		->Dict:





           '''simple docstring'''

           A__   				=					jnp.ones((batch_size, length)) / length
           return scores


    def  SCREAMING_SNAKE_CASE							(    self   :			Optional[Any])		->int:





           '''simple docstring'''

           A__   				=					None
           A__   				=					20

           A__   				=					self._get_uniform_logits(batch_size=2      ,				length=UpperCAmelCase__)

           # tweak scores to not be uniform anymore
           A__   				=					scores.at[1, 5].set((1 / length) + 0.1)  # peak, 1st batch
           A__   				=					scores.at[1, 10].set((1 / length) - 0.4)  # valley, 1st batch

           # compute softmax
           A__   				=					jax.nn.softmax(UpperCAmelCase__      ,				axis=-1)

           A__   				=					FlaxTemperatureLogitsWarper(temperature=0.5)
           A__   				=					FlaxTemperatureLogitsWarper(temperature=1.3)

           A__   				=					jax.nn.softmax(temp_dist_warper_sharper(UpperCAmelCase__      ,				scores.copy()      ,				cur_len=UpperCAmelCase__)      ,				axis=-1)
           A__   				=					jax.nn.softmax(temp_dist_warper_smoother(UpperCAmelCase__      ,				scores.copy()      ,				cur_len=UpperCAmelCase__)      ,				axis=-1)

           # uniform distribution stays uniform
           self.assertTrue(jnp.allclose(probs[0, :]      ,				warped_prob_sharp[0, :]      ,				atol=1e-3))
           self.assertTrue(jnp.allclose(probs[0, :]      ,				warped_prob_smooth[0, :]      ,				atol=1e-3))

           # sharp peaks get higher, valleys get lower
           self.assertLess(probs[1, :].max()      ,				warped_prob_sharp[1, :].max())
           self.assertGreater(probs[1, :].min()      ,				warped_prob_sharp[1, :].min())

           # smooth peaks get lower, valleys get higher
           self.assertGreater(probs[1, :].max()      ,				warped_prob_smooth[1, :].max())
           self.assertLess(probs[1, :].min()      ,				warped_prob_smooth[1, :].min())


    def  SCREAMING_SNAKE_CASE							(    self   :			str)		->int:





           '''simple docstring'''

           A__   				=					None
           A__   				=					10
           A__   				=					2

           # create ramp distribution
           A__   				=					np.broadcast_to(np.arange(UpperCAmelCase__)[None, :]      ,				(batch_size, vocab_size)).copy()
           A__   				=					ramp_logits[1:, : vocab_size // 2] + vocab_size

           A__   				=					FlaxTopKLogitsWarper(3)

           A__   				=					top_k_warp(UpperCAmelCase__      ,				UpperCAmelCase__      ,				cur_len=UpperCAmelCase__)

           # check that correct tokens are filtered
           self.assertListEqual(jnp.isinf(scores[0]).tolist()      ,				7 * [True] + 3 * [False])
           self.assertListEqual(jnp.isinf(scores[1]).tolist()      ,				2 * [True] + 3 * [False] + 5 * [True])

           # check special case
           A__   				=					5
           A__   				=					FlaxTopKLogitsWarper(top_k=1      ,				filter_value=0.0      ,				min_tokens_to_keep=3)

           A__   				=					np.broadcast_to(np.arange(UpperCAmelCase__)[None, :]      ,				(batch_size, length)).copy()
           A__   				=					top_k_warp_safety_check(UpperCAmelCase__      ,				UpperCAmelCase__      ,				cur_len=UpperCAmelCase__)

           # min_tokens overwrites k: 3 tokens are kept => 2 tokens are nullified
           self.assertListEqual((scores == 0.0).sum(axis=-1).tolist()      ,				[2, 2])


    def  SCREAMING_SNAKE_CASE							(    self   :			List[Any])		->Union[str, Any]:





           '''simple docstring'''

           A__   				=					None
           A__   				=					10
           A__   				=					2

           # create distribution and take log (inverse to Softmax as taken in TopPLogitsWarper)
           A__   				=					np.log(np.array([[0.3, 0.1, 0.1, 0.5], [0.15, 0.3, 0.3, 0.25]]))

           A__   				=					FlaxTopPLogitsWarper(0.8)
           A__   				=					np.exp(top_p_warp(UpperCAmelCase__      ,				UpperCAmelCase__      ,				cur_len=UpperCAmelCase__))

           # dist should be filtered to keep min num values so that sum is >= top_p
           # exp (-inf) => 0
           A__   				=					np.array([[0.3, 0.0, 0.0, 0.5], [0.0, 0.3, 0.3, 0.25]])
           self.assertTrue(np.allclose(UpperCAmelCase__      ,				UpperCAmelCase__      ,				atol=1e-3))

           # check edge cases with negative and extreme logits
           A__   				=					np.broadcast_to(np.arange(UpperCAmelCase__)[None, :]      ,				(batch_size, vocab_size)).copy() - (
               vocab_size // 2
           )

           # make ramp_logits more extreme
           A__   				=					ramp_logits[1] * 100.0

           # make sure at least 2 tokens are kept
           A__   				=					FlaxTopPLogitsWarper(0.9      ,				min_tokens_to_keep=2      ,				filter_value=0.0)
           A__   				=					top_p_warp(UpperCAmelCase__      ,				UpperCAmelCase__      ,				cur_len=UpperCAmelCase__)

           # first batch should keep three tokens, second batch would keep only 1, but due to `min_tokens_to_keep=2` keeps 2.
           self.assertListEqual((filtered_dist != 0.0).sum(axis=-1).tolist()      ,				[3, 2])


    def  SCREAMING_SNAKE_CASE							(    self   :			Tuple)		->Tuple:





           '''simple docstring'''

           A__   				=					20
           A__   				=					4
           A__   				=					0

           A__   				=					FlaxMinLengthLogitsProcessor(min_length=10      ,				eos_token_id=UpperCAmelCase__)

           # check that min length is applied at length 5
           A__   				=					ids_tensor((batch_size, 20)      ,				vocab_size=20)
           A__   				=					5
           A__   				=					self._get_uniform_logits(UpperCAmelCase__      ,				UpperCAmelCase__)
           A__   				=					min_dist_processor(UpperCAmelCase__      ,				UpperCAmelCase__      ,				cur_len=UpperCAmelCase__)
           self.assertListEqual(scores_before_min_length[:, eos_token_id].tolist()      ,				4 * [-float('''inf''')])

           # check that min length is not applied anymore at length 15
           A__   				=					self._get_uniform_logits(UpperCAmelCase__      ,				UpperCAmelCase__)
           A__   				=					15
           A__   				=					min_dist_processor(UpperCAmelCase__      ,				UpperCAmelCase__      ,				cur_len=UpperCAmelCase__)
           self.assertFalse(jnp.isinf(UpperCAmelCase__).any())


    def  SCREAMING_SNAKE_CASE							(    self   :			Union[str, Any])		->Union[str, Any]:





           '''simple docstring'''

           A__   				=					20
           A__   				=					4
           A__   				=					0

           A__   				=					FlaxForcedBOSTokenLogitsProcessor(bos_token_id=UpperCAmelCase__)

           # check that all scores are -inf except the bos_token_id score
           A__   				=					ids_tensor((batch_size, 1)      ,				vocab_size=20)
           A__   				=					1
           A__   				=					self._get_uniform_logits(UpperCAmelCase__      ,				UpperCAmelCase__)
           A__   				=					logits_processor(UpperCAmelCase__      ,				UpperCAmelCase__      ,				cur_len=UpperCAmelCase__)
           self.assertTrue(jnp.isneginf(scores[:, bos_token_id + 1 :]).all())
           self.assertListEqual(scores[:, bos_token_id].tolist()      ,				4 * [0])  # score for bos_token_id shold be zero

           # check that bos_token_id is not forced if current length is greater than 1
           A__   				=					3
           A__   				=					self._get_uniform_logits(UpperCAmelCase__      ,				UpperCAmelCase__)
           A__   				=					logits_processor(UpperCAmelCase__      ,				UpperCAmelCase__      ,				cur_len=UpperCAmelCase__)
           self.assertFalse(jnp.isinf(UpperCAmelCase__).any())


    def  SCREAMING_SNAKE_CASE							(    self   :			Tuple)		->str:





           '''simple docstring'''

           A__   				=					20
           A__   				=					4
           A__   				=					0
           A__   				=					5

           A__   				=					FlaxForcedEOSTokenLogitsProcessor(max_length=UpperCAmelCase__      ,				eos_token_id=UpperCAmelCase__)

           # check that all scores are -inf except the eos_token_id when max_length is reached
           A__   				=					ids_tensor((batch_size, 4)      ,				vocab_size=20)
           A__   				=					4
           A__   				=					self._get_uniform_logits(UpperCAmelCase__      ,				UpperCAmelCase__)
           A__   				=					logits_processor(UpperCAmelCase__      ,				UpperCAmelCase__      ,				cur_len=UpperCAmelCase__)
           self.assertTrue(jnp.isneginf(scores[:, eos_token_id + 1 :]).all())
           self.assertListEqual(scores[:, eos_token_id].tolist()      ,				4 * [0])  # score for eos_token_id should be zero

           # check that eos_token_id is not forced if max_length is not reached
           A__   				=					3
           A__   				=					self._get_uniform_logits(UpperCAmelCase__      ,				UpperCAmelCase__)
           A__   				=					logits_processor(UpperCAmelCase__      ,				UpperCAmelCase__      ,				cur_len=UpperCAmelCase__)
           self.assertFalse(jnp.isinf(UpperCAmelCase__).any())


    def  SCREAMING_SNAKE_CASE							(    self   :			List[str])		->Tuple:





           '''simple docstring'''

           A__   				=					4
           A__   				=					10
           A__   				=					15
           A__   				=					2
           A__   				=					1
           A__   				=					15

           # dummy input_ids and scores
           A__   				=					ids_tensor((batch_size, sequence_length)      ,				UpperCAmelCase__)
           A__   				=					input_ids.copy()

           A__   				=					self._get_uniform_logits(UpperCAmelCase__      ,				UpperCAmelCase__)
           A__   				=					scores.copy()

           # instantiate all dist processors
           A__   				=					FlaxTemperatureLogitsWarper(temperature=0.5)
           A__   				=					FlaxTopKLogitsWarper(3)
           A__   				=					FlaxTopPLogitsWarper(0.8)

           # instantiate all logits processors
           A__   				=					FlaxMinLengthLogitsProcessor(min_length=10      ,				eos_token_id=UpperCAmelCase__)
           A__   				=					FlaxForcedBOSTokenLogitsProcessor(bos_token_id=UpperCAmelCase__)
           A__   				=					FlaxForcedEOSTokenLogitsProcessor(max_length=UpperCAmelCase__      ,				eos_token_id=UpperCAmelCase__)

           A__   				=					10

           # no processor list
           A__   				=					temp_dist_warp(UpperCAmelCase__      ,				UpperCAmelCase__      ,				cur_len=UpperCAmelCase__)
           A__   				=					top_k_warp(UpperCAmelCase__      ,				UpperCAmelCase__      ,				cur_len=UpperCAmelCase__)
           A__   				=					top_p_warp(UpperCAmelCase__      ,				UpperCAmelCase__      ,				cur_len=UpperCAmelCase__)
           A__   				=					min_dist_proc(UpperCAmelCase__      ,				UpperCAmelCase__      ,				cur_len=UpperCAmelCase__)
           A__   				=					bos_dist_proc(UpperCAmelCase__      ,				UpperCAmelCase__      ,				cur_len=UpperCAmelCase__)
           A__   				=					eos_dist_proc(UpperCAmelCase__      ,				UpperCAmelCase__      ,				cur_len=UpperCAmelCase__)

           # with processor list
           A__   				=					FlaxLogitsProcessorList(
               [temp_dist_warp, top_k_warp, top_p_warp, min_dist_proc, bos_dist_proc, eos_dist_proc])
           A__   				=					processor(UpperCAmelCase__      ,				UpperCAmelCase__      ,				cur_len=UpperCAmelCase__)

           # scores should be equal
           self.assertTrue(jnp.allclose(UpperCAmelCase__      ,				UpperCAmelCase__      ,				atol=1e-3))

           # input_ids should never be changed
           self.assertListEqual(input_ids.tolist()      ,				input_ids_comp.tolist())





    def  SCREAMING_SNAKE_CASE							(    self   :			Union[str, Any])		->int:





           '''simple docstring'''

           A__   				=					4
           A__   				=					10
           A__   				=					15
           A__   				=					2
           A__   				=					1
           A__   				=					15

           # dummy input_ids and scores
           A__   				=					ids_tensor((batch_size, sequence_length)      ,				UpperCAmelCase__)
           A__   				=					input_ids.copy()

           A__   				=					self._get_uniform_logits(UpperCAmelCase__      ,				UpperCAmelCase__)
           A__   				=					scores.copy()

           # instantiate all dist processors
           A__   				=					FlaxTemperatureLogitsWarper(temperature=0.5)
           A__   				=					FlaxTopKLogitsWarper(3)
           A__   				=					FlaxTopPLogitsWarper(0.8)

           # instantiate all logits processors
           A__   				=					FlaxMinLengthLogitsProcessor(min_length=10      ,				eos_token_id=UpperCAmelCase__)
           A__   				=					FlaxForcedBOSTokenLogitsProcessor(bos_token_id=UpperCAmelCase__)
           A__   				=					FlaxForcedEOSTokenLogitsProcessor(max_length=UpperCAmelCase__      ,				eos_token_id=UpperCAmelCase__)

           A__   				=					10

           # no processor list
           def run_no_processor_list(UpperCAmelCase__   :			Dict      ,				UpperCAmelCase__   :			Optional[int]      ,				UpperCAmelCase__   :			Union[str, Any]):
                  A__   				=					temp_dist_warp(UpperCAmelCase__      ,				UpperCAmelCase__      ,				cur_len=UpperCAmelCase__)
                  A__   				=					top_k_warp(UpperCAmelCase__      ,				UpperCAmelCase__      ,				cur_len=UpperCAmelCase__)
                  A__   				=					top_p_warp(UpperCAmelCase__      ,				UpperCAmelCase__      ,				cur_len=UpperCAmelCase__)
                  A__   				=					min_dist_proc(UpperCAmelCase__      ,				UpperCAmelCase__      ,				cur_len=UpperCAmelCase__)
                  A__   				=					bos_dist_proc(UpperCAmelCase__      ,				UpperCAmelCase__      ,				cur_len=UpperCAmelCase__)
                  A__   				=					eos_dist_proc(UpperCAmelCase__      ,				UpperCAmelCase__      ,				cur_len=UpperCAmelCase__)
                  return scores

           # with processor list
           def run_processor_list(UpperCAmelCase__   :			Any      ,				UpperCAmelCase__   :			Dict      ,				UpperCAmelCase__   :			Optional[Any]):
                  A__   				=					FlaxLogitsProcessorList(
                      [temp_dist_warp, top_k_warp, top_p_warp, min_dist_proc, bos_dist_proc, eos_dist_proc])
                  A__   				=					processor(UpperCAmelCase__      ,				UpperCAmelCase__      ,				cur_len=UpperCAmelCase__)
                  return scores

           A__   				=					jax.jit(UpperCAmelCase__)
           A__   				=					jax.jit(UpperCAmelCase__)

           A__   				=					jitted_run_no_processor_list(UpperCAmelCase__      ,				UpperCAmelCase__      ,				UpperCAmelCase__)
           A__   				=					jitted_run_processor_list(UpperCAmelCase__      ,				UpperCAmelCase__      ,				UpperCAmelCase__)

           # scores should be equal
           self.assertTrue(jnp.allclose(UpperCAmelCase__      ,				UpperCAmelCase__      ,				atol=1e-3))

           # input_ids should never be changed
           self.assertListEqual(input_ids.tolist()      ,				input_ids_comp.tolist())


