

import unittest

from transformers import (
    MODEL_FOR_CAUSAL_LM_MAPPING,
    TF_MODEL_FOR_CAUSAL_LM_MAPPING,
    TextGenerationPipeline,
    logging,
    pipeline,
)
from transformers.testing_utils import (
    CaptureLogger,
    is_pipeline_test,
    require_accelerate,
    require_tf,
    require_torch,
    require_torch_gpu,
    require_torch_or_tf,
)

from .test_pipelines_common import ANY

@is_pipeline_test
@require_torch_or_tf
class    _lowercase				(					unittest.TestCase			):
   _lowercase     :       Dict  					=    MODEL_FOR_CAUSAL_LM_MAPPING
   _lowercase     :       str  					=    TF_MODEL_FOR_CAUSAL_LM_MAPPING

   @require_torch
   def     UpperCamelCase					(				self	:    Optional[Any]       ) ->			List[Any]:
         """simple docstring"""





         A_							       =		pipeline(task='''text-generation'''			,						model='''sshleifer/tiny-ctrl'''			,						framework='''pt'''       )
         # Using `do_sample=False` to force deterministic output
         A_							       =		text_generator('''This is a test'''			,						do_sample=lowerCamelCase__       )
         self.assertEqual(
             lowerCamelCase__			,						[
                 {
                     '''generated_text''': (
                         '''This is a test ☃ ☃ segmental segmental segmental 议议eski eski flutter flutter Lacy oscope.'''
                         ''' oscope. FiliFili@@'''
                     )
                 }
             ]			,						)

         A_							       =		text_generator(['''This is a test''', '''This is a second test''']       )
         self.assertEqual(
             lowerCamelCase__			,						[
                 [
                     {
                         '''generated_text''': (
                             '''This is a test ☃ ☃ segmental segmental segmental 议议eski eski flutter flutter Lacy oscope.'''
                             ''' oscope. FiliFili@@'''
                         )
                     }
                 ],
                 [
                     {
                         '''generated_text''': (
                             '''This is a second test ☃ segmental segmental segmental 议议eski eski flutter flutter Lacy'''
                             ''' oscope. oscope. FiliFili@@'''
                         )
                     }
                 ],
             ]			,						)

         A_							       =		text_generator('''This is a test'''			,						do_sample=lowerCamelCase__			,						num_return_sequences=2			,						return_tensors=lowerCamelCase__       )
         self.assertEqual(
             lowerCamelCase__			,						[
                 {'''generated_token_ids''': ANY(lowerCamelCase__       )},
                 {'''generated_token_ids''': ANY(lowerCamelCase__       )},
             ]			,						)
         A_							       =		text_generator.model.config.eos_token_id
         A_							       =		'''<pad>'''
         A_							       =		text_generator(
             ['''This is a test''', '''This is a second test''']			,						do_sample=lowerCamelCase__			,						num_return_sequences=2			,						batch_size=2			,						return_tensors=lowerCamelCase__			,						)
         self.assertEqual(
             lowerCamelCase__			,						[
                 [
                     {'''generated_token_ids''': ANY(lowerCamelCase__       )},
                     {'''generated_token_ids''': ANY(lowerCamelCase__       )},
                 ],
                 [
                     {'''generated_token_ids''': ANY(lowerCamelCase__       )},
                     {'''generated_token_ids''': ANY(lowerCamelCase__       )},
                 ],
             ]			,						)

   @require_tf
   def     UpperCamelCase					(				self	:    Dict       ) ->			List[str]:
         """simple docstring"""





         A_							       =		pipeline(task='''text-generation'''			,						model='''sshleifer/tiny-ctrl'''			,						framework='''tf'''       )

         # Using `do_sample=False` to force deterministic output
         A_							       =		text_generator('''This is a test'''			,						do_sample=lowerCamelCase__       )
         self.assertEqual(
             lowerCamelCase__			,						[
                 {
                     '''generated_text''': (
                         '''This is a test FeyFeyFey(Croatis.), s.), Cannes Cannes Cannes 閲閲Cannes Cannes Cannes 攵'''
                         ''' please,'''
                     )
                 }
             ]			,						)

         A_							       =		text_generator(['''This is a test''', '''This is a second test''']			,						do_sample=lowerCamelCase__       )
         self.assertEqual(
             lowerCamelCase__			,						[
                 [
                     {
                         '''generated_text''': (
                             '''This is a test FeyFeyFey(Croatis.), s.), Cannes Cannes Cannes 閲閲Cannes Cannes Cannes 攵'''
                             ''' please,'''
                         )
                     }
                 ],
                 [
                     {
                         '''generated_text''': (
                             '''This is a second test Chieftain Chieftain prefecture prefecture prefecture Cannes Cannes'''
                             ''' Cannes 閲閲Cannes Cannes Cannes 攵 please,'''
                         )
                     }
                 ],
             ]			,						)

   def     UpperCamelCase					(				self	:    Dict			,						lowerCamelCase__	:    Dict			,						lowerCamelCase__	:    Optional[Any]			,						lowerCamelCase__	:    List[str]       ) ->			Union[str, Any]:
         """simple docstring"""





         A_							       =		TextGenerationPipeline(model=lowerCamelCase__			,						tokenizer=lowerCamelCase__       )
         return text_generator, ["This is a test", "Another test"]

   def     UpperCamelCase					(				self	:    Union[str, Any]       ) ->			List[str]:
         """simple docstring"""





         A_							       =		'''Hello I believe in'''
         A_							       =		pipeline('''text-generation'''			,						model='''hf-internal-testing/tiny-random-gpt2'''       )
         A_							       =		text_generator(lowerCamelCase__       )
         self.assertEqual(
             lowerCamelCase__			,						[{'''generated_text''': '''Hello I believe in fe fe fe fe fe fe fe fe fe fe fe fe'''}]			,						)

         A_							       =		text_generator(lowerCamelCase__			,						stop_sequence=''' fe'''       )
         self.assertEqual(lowerCamelCase__			,						[{'''generated_text''': '''Hello I believe in fe'''}]       )

   def     UpperCamelCase					(				self	:    Tuple			,						lowerCamelCase__	:    str			,						lowerCamelCase__	:    List[str]       ) ->			str:
         """simple docstring"""





         A_							       =		text_generator.model
         A_							       =		text_generator.tokenizer

         A_							       =		text_generator('''This is a test'''       )
         self.assertEqual(lowerCamelCase__			,						[{'''generated_text''': ANY(lowerCamelCase__       )}]       )
         self.assertTrue(outputs[0]['''generated_text'''].startswith('''This is a test'''       )       )

         A_							       =		text_generator('''This is a test'''			,						return_full_text=lowerCamelCase__       )
         self.assertEqual(lowerCamelCase__			,						[{'''generated_text''': ANY(lowerCamelCase__       )}]       )
         self.assertNotIn('''This is a test'''			,						outputs[0]['''generated_text''']       )

         A_							       =		pipeline(task='''text-generation'''			,						model=lowerCamelCase__			,						tokenizer=lowerCamelCase__			,						return_full_text=lowerCamelCase__       )
         A_							       =		text_generator('''This is a test'''       )
         self.assertEqual(lowerCamelCase__			,						[{'''generated_text''': ANY(lowerCamelCase__       )}]       )
         self.assertNotIn('''This is a test'''			,						outputs[0]['''generated_text''']       )

         A_							       =		text_generator('''This is a test'''			,						return_full_text=lowerCamelCase__       )
         self.assertEqual(lowerCamelCase__			,						[{'''generated_text''': ANY(lowerCamelCase__       )}]       )
         self.assertTrue(outputs[0]['''generated_text'''].startswith('''This is a test'''       )       )

         A_							       =		text_generator(['''This is great !''', '''Something else''']			,						num_return_sequences=2			,						do_sample=lowerCamelCase__       )
         self.assertEqual(
             lowerCamelCase__			,						[
                 [{'''generated_text''': ANY(lowerCamelCase__       )}, {'''generated_text''': ANY(lowerCamelCase__       )}],
                 [{'''generated_text''': ANY(lowerCamelCase__       )}, {'''generated_text''': ANY(lowerCamelCase__       )}],
             ]			,						)

         if text_generator.tokenizer.pad_token is not None:
               A_							       =		text_generator(
                   ['''This is great !''', '''Something else''']			,						num_return_sequences=2			,						batch_size=2			,						do_sample=lowerCamelCase__       )
               self.assertEqual(
                   lowerCamelCase__			,						[
                       [{'''generated_text''': ANY(lowerCamelCase__       )}, {'''generated_text''': ANY(lowerCamelCase__       )}],
                       [{'''generated_text''': ANY(lowerCamelCase__       )}, {'''generated_text''': ANY(lowerCamelCase__       )}],
                   ]			,						)

         with self.assertRaises(lowerCamelCase__       ):
               A_							       =		text_generator('''test'''			,						return_full_text=lowerCamelCase__			,						return_text=lowerCamelCase__       )
         with self.assertRaises(lowerCamelCase__       ):
               A_							       =		text_generator('''test'''			,						return_full_text=lowerCamelCase__			,						return_tensors=lowerCamelCase__       )
         with self.assertRaises(lowerCamelCase__       ):
               A_							       =		text_generator('''test'''			,						return_text=lowerCamelCase__			,						return_tensors=lowerCamelCase__       )

         # Empty prompt is slighly special
         # it requires BOS token to exist.
         # Special case for Pegasus which will always append EOS so will
         # work even without BOS.
         if (
             text_generator.tokenizer.bos_token_id is not None
             or "Pegasus" in tokenizer.__class__.__name__
             or "Git" in model.__class__.__name__
         ):
               A_							       =		text_generator(''''''       )
               self.assertEqual(lowerCamelCase__			,						[{'''generated_text''': ANY(lowerCamelCase__       )}]       )
         else:
               with self.assertRaises((ValueError, AssertionError)       ):
                     A_							       =		text_generator(''''''       )

         if text_generator.framework == "tf":
               # TF generation does not support max_new_tokens, and it's impossible
               # to control long generation with only max_length without
               # fancy calculation, dismissing tests for now.
               return
         # We don't care about infinite range models.
         # They already work.
         # Skip this test for XGLM, since it uses sinusoidal positional embeddings which are resized on-the-fly.
         A_							       =		['''RwkvForCausalLM''', '''XGLMForCausalLM''', '''GPTNeoXForCausalLM''']
         if (
             tokenizer.model_max_length < 1_0_0_0_0
             and text_generator.model.__class__.__name__ not in EXTRA_MODELS_CAN_HANDLE_LONG_INPUTS
         ):
               # Handling of large generations
               with self.assertRaises((RuntimeError, IndexError, ValueError, AssertionError)       ):
                     text_generator('''This is a test''' * 5_0_0			,						max_new_tokens=2_0       )

               A_							       =		text_generator('''This is a test''' * 5_0_0			,						handle_long_generation='''hole'''			,						max_new_tokens=2_0       )
               # Hole strategy cannot work
               with self.assertRaises(lowerCamelCase__       ):
                     text_generator(
                         '''This is a test''' * 5_0_0			,						handle_long_generation='''hole'''			,						max_new_tokens=tokenizer.model_max_length + 1_0			,						)

   @require_torch
   @require_accelerate
   @require_torch_gpu
   def     UpperCamelCase					(				self	:    Optional[Any]       ) ->			Dict:
         """simple docstring"""





         import torch

         # Classic `model_kwargs`
         A_							       =		pipeline(
             model='''hf-internal-testing/tiny-random-bloom'''			,						model_kwargs={'''device_map''': '''auto''', '''torch_dtype''': torch.bfloataa}			,						)
         self.assertEqual(pipe.model.device			,						torch.device(0       )       )
         self.assertEqual(pipe.model.lm_head.weight.dtype			,						torch.bfloataa       )
         A_							       =		pipe('''This is a test'''       )
         self.assertEqual(
             lowerCamelCase__			,						[
                 {
                     '''generated_text''': (
                         '''This is a test test test test test test test test test test test test test test test test'''
                         ''' test'''
                     )
                 }
             ]			,						)

         # Upgraded those two to real pipeline arguments (they just get sent for the model as they're unlikely to mean anything else.)
         A_							       =		pipeline(model='''hf-internal-testing/tiny-random-bloom'''			,						device_map='''auto'''			,						torch_dtype=torch.bfloataa       )
         self.assertEqual(pipe.model.device			,						torch.device(0       )       )
         self.assertEqual(pipe.model.lm_head.weight.dtype			,						torch.bfloataa       )
         A_							       =		pipe('''This is a test'''       )
         self.assertEqual(
             lowerCamelCase__			,						[
                 {
                     '''generated_text''': (
                         '''This is a test test test test test test test test test test test test test test test test'''
                         ''' test'''
                     )
                 }
             ]			,						)

         # torch_dtype will be automatically set to float32 if not provided - check: https://github.com/huggingface/transformers/pull/20602
         A_							       =		pipeline(model='''hf-internal-testing/tiny-random-bloom'''			,						device_map='''auto'''       )
         self.assertEqual(pipe.model.device			,						torch.device(0       )       )
         self.assertEqual(pipe.model.lm_head.weight.dtype			,						torch.floataa       )
         A_							       =		pipe('''This is a test'''       )
         self.assertEqual(
             lowerCamelCase__			,						[
                 {
                     '''generated_text''': (
                         '''This is a test test test test test test test test test test test test test test test test'''
                         ''' test'''
                     )
                 }
             ]			,						)

   @require_torch
   @require_torch_gpu
   def     UpperCamelCase					(				self	:    int       ) ->			Any:
         """simple docstring"""





         import torch

         A_							       =		pipeline(model='''hf-internal-testing/tiny-random-bloom'''			,						device=0			,						torch_dtype=torch.floataa       )
         pipe('''This is a test'''       )

   @require_torch
   @require_accelerate
   @require_torch_gpu
   def     UpperCamelCase					(				self	:    str       ) ->			Optional[int]:
         """simple docstring"""





         import torch

         A_							       =		pipeline(model='''hf-internal-testing/tiny-random-bloom'''			,						device_map='''auto'''			,						torch_dtype=torch.floataa       )
         pipe('''This is a test'''			,						do_sample=lowerCamelCase__			,						top_p=0.5       )





   def     UpperCamelCase					(				self	:    Any       ) ->			str:
         """simple docstring"""





         A_							       =		'''Hello world'''
         A_							       =		pipeline('''text-generation'''			,						model='''hf-internal-testing/tiny-random-gpt2'''       )
         if text_generator.model.framework == "tf":
               A_							       =		logging.get_logger('''transformers.generation.tf_utils'''       )
         else:
               A_							       =		logging.get_logger('''transformers.generation.utils'''       )
         A_							       =		'''Both `max_new_tokens`'''  # The beggining of the message to be checked in this test

         # Both are set by the user -> log warning
         with CaptureLogger(lowerCamelCase__       ) as cl:
               A_							       =		text_generator(lowerCamelCase__			,						max_length=1_0			,						max_new_tokens=1       )
         self.assertIn(lowerCamelCase__			,						cl.out       )

         # The user only sets one -> no warning
         with CaptureLogger(lowerCamelCase__       ) as cl:
               A_							       =		text_generator(lowerCamelCase__			,						max_new_tokens=1       )
         self.assertNotIn(lowerCamelCase__			,						cl.out       )

         with CaptureLogger(lowerCamelCase__       ) as cl:
               A_							       =		text_generator(lowerCamelCase__			,						max_length=1_0       )
         self.assertNotIn(lowerCamelCase__			,						cl.out       )








import json
import os
import re
import shutil
import tempfile
import unittest
from typing import Tuple

from transformers import AddedToken, BatchEncoding, PerceiverTokenizer
from transformers.utils import cached_property, is_tf_available, is_torch_available

from ...test_tokenization_common import TokenizerTesterMixin


if is_torch_available():
      __lowercase				      =						"""pt"""
elif is_tf_available():
      __lowercase				      =						"""tf"""
else:
      __lowercase				      =						"""jax"""

class    _lowercase				(					__lowerCamelCase,unittest.TestCase			):
   _lowercase     :       Dict  					=    PerceiverTokenizer
   _lowercase     :       Dict  					=    False

   def     UpperCamelCase					(				self	:    Tuple       ) ->			Any:
         """simple docstring"""





         super().setUp()
         A_							       =		PerceiverTokenizer()
         tokenizer.save_pretrained(self.tmpdirname       )

   @cached_property
   def     UpperCamelCase					(				self	:    Tuple       ) ->			List[str]:
         """simple docstring"""





         return PerceiverTokenizer.from_pretrained('''deepmind/language-perceiver'''       )

   def     UpperCamelCase					(				self	:    List[str]			,						**lowerCamelCase__	:    str       ) ->			PerceiverTokenizer:
         """simple docstring"""





         return self.tokenizer_class.from_pretrained(self.tmpdirname			,						**lowerCamelCase__       )

   def     UpperCamelCase					(				self	:    List[str]			,						lowerCamelCase__	:    str			,						lowerCamelCase__	:    Optional[Any]=False			,						lowerCamelCase__	:    List[str]=2_0			,						lowerCamelCase__	:    Union[str, Any]=5       ) ->			Tuple[str, list]:
         """simple docstring"""





         A_							       =		[]
         for i in range(len(lowerCamelCase__       )       ):
               try:
                     A_							       =		tokenizer.decode([i]			,						clean_up_tokenization_spaces=lowerCamelCase__       )
               except UnicodeDecodeError:
                     pass
               toks.append((i, tok)       )

         A_							       =		list(filter(lambda lowerCamelCase__       : re.match(R'''^[ a-zA-Z]+$'''			,						t[1]       )			,						lowerCamelCase__       )       )
         A_							       =		list(filter(lambda lowerCamelCase__       : [t[0]] == tokenizer.encode(t[1]			,						add_special_tokens=lowerCamelCase__       )			,						lowerCamelCase__       )       )
         if max_length is not None and len(lowerCamelCase__       ) > max_length:
               A_							       =		toks[:max_length]
         if min_length is not None and len(lowerCamelCase__       ) < min_length and len(lowerCamelCase__       ) > 0:
               while len(lowerCamelCase__       ) < min_length:
                     A_							       =		toks + toks
        # toks_str = [t[1] for t in toks]
         A_							       =		[t[0] for t in toks]

         # Ensure consistency
         A_							       =		tokenizer.decode(lowerCamelCase__			,						clean_up_tokenization_spaces=lowerCamelCase__       )
         if " " not in output_txt and len(lowerCamelCase__       ) > 1:
               A_							       =		(
                   tokenizer.decode([toks_ids[0]]			,						clean_up_tokenization_spaces=lowerCamelCase__       )
                   + ''' '''
                   + tokenizer.decode(toks_ids[1:]			,						clean_up_tokenization_spaces=lowerCamelCase__       )
               )
         if with_prefix_space:
               A_							       =		''' ''' + output_txt
         A_							       =		tokenizer.encode(lowerCamelCase__			,						add_special_tokens=lowerCamelCase__       )
         return output_txt, output_ids

   def     UpperCamelCase					(				self	:    Optional[Any]       ) ->			List[str]:
         """simple docstring"""





         A_							       =		self.perceiver_tokenizer
         A_							       =		'''Unicode €.'''
         A_							       =		tokenizer(lowerCamelCase__       )
         A_							       =		[4, 9_1, 1_1_6, 1_1_1, 1_0_5, 1_1_7, 1_0_6, 1_0_7, 3_8, 2_3_2, 1_3_6, 1_7_8, 5_2, 5]
         self.assertEqual(encoded['''input_ids''']			,						lowerCamelCase__       )

         # decoding
         A_							       =		tokenizer.decode(lowerCamelCase__       )
         self.assertEqual(lowerCamelCase__			,						'''[CLS]Unicode €.[SEP]'''       )

         A_							       =		tokenizer('''e è é ê ë'''       )
         A_							       =		[4, 1_0_7, 3_8, 2_0_1, 1_7_4, 3_8, 2_0_1, 1_7_5, 3_8, 2_0_1, 1_7_6, 3_8, 2_0_1, 1_7_7, 5]
         self.assertEqual(encoded['''input_ids''']			,						lowerCamelCase__       )
         # decoding
         A_							       =		tokenizer.decode(lowerCamelCase__       )
         self.assertEqual(lowerCamelCase__			,						'''[CLS]e è é ê ë[SEP]'''       )

         # encode/decode, but with `encode` instead of `__call__`
         self.assertEqual(tokenizer.decode(tokenizer.encode('''e è é ê ë'''       )       )			,						'''[CLS]e è é ê ë[SEP]'''       )

   def     UpperCamelCase					(				self	:    Union[str, Any]       ) ->			Union[str, Any]:
         """simple docstring"""





         A_							       =		self.perceiver_tokenizer
         A_							       =		['''A long paragraph for summarization.''', '''Another paragraph for summarization.''']
         # fmt: off
         A_							       =		[4, 7_1, 3_8, 1_1_4, 1_1_7, 1_1_6, 1_0_9, 3_8, 1_1_8, 1_0_3, 1_2_0, 1_0_3, 1_0_9, 1_2_0, 1_0_3, 1_1_8, 1_1_0, 3_8, 1_0_8, 1_1_7, 1_2_0, 3_8, 1_2_1, 1_2_3, 1_1_5, 1_1_5, 1_0_3, 1_2_0, 1_1_1, 1_2_8, 1_0_3, 1_2_2, 1_1_1, 1_1_7, 1_1_6, 5_2, 5, 0]
         # fmt: on
         A_							       =		tokenizer(lowerCamelCase__			,						padding=lowerCamelCase__			,						return_tensors=lowerCamelCase__       )
         self.assertIsInstance(lowerCamelCase__			,						lowerCamelCase__       )

         if FRAMEWORK != "jax":
               A_							       =		list(batch.input_ids.numpy()[0]       )
         else:
               A_							       =		list(batch.input_ids.tolist()[0]       )

         self.assertListEqual(lowerCamelCase__			,						lowerCamelCase__       )

         self.assertEqual((2, 3_8)			,						batch.input_ids.shape       )
         self.assertEqual((2, 3_8)			,						batch.attention_mask.shape       )

   def     UpperCamelCase					(				self	:    str       ) ->			List[str]:
         """simple docstring"""





         A_							       =		self.perceiver_tokenizer
         A_							       =		['''A long paragraph for summarization.''', '''Another paragraph for summarization.''']
         A_							       =		tokenizer(lowerCamelCase__			,						padding=lowerCamelCase__			,						return_tensors=lowerCamelCase__       )
         # check if input_ids are returned and no decoder_input_ids
         self.assertIn('''input_ids'''			,						lowerCamelCase__       )
         self.assertIn('''attention_mask'''			,						lowerCamelCase__       )
         self.assertNotIn('''decoder_input_ids'''			,						lowerCamelCase__       )
         self.assertNotIn('''decoder_attention_mask'''			,						lowerCamelCase__       )

   def     UpperCamelCase					(				self	:    Any       ) ->			Union[str, Any]:
         """simple docstring"""





         A_							       =		self.perceiver_tokenizer
         A_							       =		[
             '''Summary of the text.''',
             '''Another summary.''',
         ]
         A_							       =		tokenizer(
             text_target=lowerCamelCase__			,						max_length=3_2			,						padding='''max_length'''			,						truncation=lowerCamelCase__			,						return_tensors=lowerCamelCase__       )
         self.assertEqual(3_2			,						targets['''input_ids'''].shape[1]       )

   def     UpperCamelCase					(				self	:    Dict       ) ->			List[str]:
         """simple docstring"""





         A_							       =		self.get_tokenizers()
         for tokenizer in tokenizers:
               with self.subTest(F"{tokenizer.__class__.__name__}"       ):
                     self.assertNotEqual(tokenizer.model_max_length			,						4_2       )

        # Now let's start the test
         A_							       =		self.get_tokenizers()
         for tokenizer in tokenizers:
               with self.subTest(F"{tokenizer.__class__.__name__}"       ):
                     # Isolate this from the other tests because we save additional tokens/etc
                     A_							       =		tempfile.mkdtemp()

                     A_							       =		''' He is very happy, UNwant\u00E9d,running'''
                     A_							       =		tokenizer.encode(lowerCamelCase__			,						add_special_tokens=lowerCamelCase__       )
                     tokenizer.save_pretrained(lowerCamelCase__       )

                     A_							       =		tokenizer.__class__.from_pretrained(lowerCamelCase__       )
                     A_							       =		after_tokenizer.encode(lowerCamelCase__			,						add_special_tokens=lowerCamelCase__       )
                     self.assertListEqual(lowerCamelCase__			,						lowerCamelCase__       )

                     shutil.rmtree(lowerCamelCase__       )

         A_							       =		self.get_tokenizers(model_max_length=4_2       )
         for tokenizer in tokenizers:
               with self.subTest(F"{tokenizer.__class__.__name__}"       ):
                     # Isolate this from the other tests because we save additional tokens/etc
                     A_							       =		tempfile.mkdtemp()

                     A_							       =		''' He is very happy, UNwant\u00E9d,running'''
                     tokenizer.add_tokens(['''bim''', '''bambam''']       )
                     A_							       =		tokenizer.additional_special_tokens
                     additional_special_tokens.append('''new_additional_special_token'''       )
                     tokenizer.add_special_tokens({'''additional_special_tokens''': additional_special_tokens}       )
                     A_							       =		tokenizer.encode(lowerCamelCase__			,						add_special_tokens=lowerCamelCase__       )
                     tokenizer.save_pretrained(lowerCamelCase__       )

                     A_							       =		tokenizer.__class__.from_pretrained(lowerCamelCase__       )
                     A_							       =		after_tokenizer.encode(lowerCamelCase__			,						add_special_tokens=lowerCamelCase__       )
                     self.assertListEqual(lowerCamelCase__			,						lowerCamelCase__       )
                     self.assertIn('''new_additional_special_token'''			,						after_tokenizer.additional_special_tokens       )
                     self.assertEqual(after_tokenizer.model_max_length			,						4_2       )

                     A_							       =		tokenizer.__class__.from_pretrained(lowerCamelCase__			,						model_max_length=4_3       )
                     self.assertEqual(tokenizer.model_max_length			,						4_3       )

                     shutil.rmtree(lowerCamelCase__       )

   def     UpperCamelCase					(				self	:    List[Any]       ) ->			Any:
         """simple docstring"""





         A_							       =		[]
         if self.test_slow_tokenizer:
               tokenizer_list.append((self.tokenizer_class, self.get_tokenizer())       )

         if self.test_rust_tokenizer:
               tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer())       )

         for tokenizer_class, tokenizer_utils in tokenizer_list:
               with tempfile.TemporaryDirectory() as tmp_dir:
                     tokenizer_utils.save_pretrained(lowerCamelCase__       )

                     with open(os.path.join(lowerCamelCase__			,						'''special_tokens_map.json'''       )			,						encoding='''utf-8'''       ) as json_file:
                           A_							       =		json.load(lowerCamelCase__       )

                     with open(os.path.join(lowerCamelCase__			,						'''tokenizer_config.json'''       )			,						encoding='''utf-8'''       ) as json_file:
                           A_							       =		json.load(lowerCamelCase__       )

                     A_							       =		[F"<extra_id_{i}>" for i in range(1_2_5       )]

                     A_							       =		added_tokens_extra_ids + [
                         '''an_additional_special_token'''
                     ]
                     A_							       =		added_tokens_extra_ids + [
                         '''an_additional_special_token'''
                     ]

                     with open(os.path.join(lowerCamelCase__			,						'''special_tokens_map.json'''       )			,						'''w'''			,						encoding='''utf-8'''       ) as outfile:
                           json.dump(lowerCamelCase__			,						lowerCamelCase__       )
                     with open(os.path.join(lowerCamelCase__			,						'''tokenizer_config.json'''       )			,						'''w'''			,						encoding='''utf-8'''       ) as outfile:
                           json.dump(lowerCamelCase__			,						lowerCamelCase__       )

                     # the following checks allow us to verify that our test works as expected, i.e. that the tokenizer takes
                     # into account the new value of additional_special_tokens given in the "tokenizer_config.json" and
                     # "special_tokens_map.json" files
                     A_							       =		tokenizer_class.from_pretrained(
                         lowerCamelCase__			,						)
                     self.assertIn(
                         '''an_additional_special_token'''			,						tokenizer_without_change_in_init.additional_special_tokens       )
                     self.assertEqual(
                         ['''an_additional_special_token''']			,						tokenizer_without_change_in_init.convert_ids_to_tokens(
                             tokenizer_without_change_in_init.convert_tokens_to_ids(['''an_additional_special_token''']       )       )			,						)

                     # Now we test that we can change the value of additional_special_tokens in the from_pretrained
                     A_							       =		added_tokens_extra_ids + [AddedToken('''a_new_additional_special_token'''			,						lstrip=lowerCamelCase__       )]
                     A_							       =		tokenizer_class.from_pretrained(
                         lowerCamelCase__			,						additional_special_tokens=lowerCamelCase__			,						)

                     self.assertIn('''a_new_additional_special_token'''			,						tokenizer.additional_special_tokens       )
                     self.assertEqual(
                         ['''a_new_additional_special_token''']			,						tokenizer.convert_ids_to_tokens(
                             tokenizer.convert_tokens_to_ids(['''a_new_additional_special_token''']       )       )			,						)

   def     UpperCamelCase					(				self	:    Dict       ) ->			List[Any]:
         """simple docstring"""





         A_							       =		self.perceiver_tokenizer
         self.assertEqual(tokenizer.decode([1_7_8]       )			,						'''�'''       )

   def     UpperCamelCase					(				self	:    int       ) ->			Any:
         """simple docstring"""





         pass

   def     UpperCamelCase					(				self	:    int       ) ->			int:
         """simple docstring"""





         pass

   def     UpperCamelCase					(				self	:    List[str]       ) ->			Dict:
         """simple docstring"""





         pass

   def     UpperCamelCase					(				self	:    int       ) ->			Any:
         """simple docstring"""





         pass





   def     UpperCamelCase					(				self	:    List[Any]       ) ->			Dict:
         """simple docstring"""





         A_							       =		self.get_tokenizers(fast=lowerCamelCase__			,						do_lower_case=lowerCamelCase__       )
         for tokenizer in tokenizers:
               with self.subTest(F"{tokenizer.__class__.__name__}"       ):
                     A_							       =		['''[CLS]''', '''t''', '''h''', '''i''', '''s''', ''' ''', '''i''', '''s''', ''' ''', '''a''', ''' ''', '''t''', '''e''', '''s''', '''t''', '''[SEP]''']
                     A_							       =		tokenizer.convert_tokens_to_string(lowerCamelCase__       )

                     self.assertIsInstance(lowerCamelCase__			,						lowerCamelCase__       )





