





from typing import Any, Dict, List, Optional, Tuple, Union

import torch
from torch import nn
from torch.utils.data import DistributedSampler, RandomSampler

from transformers import PreTrainedModel, Trainer, logging
from transformers.integrations import is_fairscale_available
from transformers.models.fsmt.configuration_fsmt import FSMTConfig
from transformers.optimization import (
    Adafactor,
    AdamW,
    get_constant_schedule,
    get_constant_schedule_with_warmup,
    get_cosine_schedule_with_warmup,
    get_cosine_with_hard_restarts_schedule_with_warmup,
    get_linear_schedule_with_warmup,
    get_polynomial_decay_schedule_with_warmup,
)
from transformers.trainer_pt_utils import get_tpu_sampler
from transformers.training_args import ParallelMode
from transformers.utils import is_torch_tpu_available


if is_fairscale_available():
     from fairscale.optim import OSS


__lowercase	: Optional[Any]       =logging.get_logger(__name__)

__lowercase	: str       ={
    """linear""": get_linear_schedule_with_warmup,
    """cosine""": get_cosine_schedule_with_warmup,
    """cosine_w_restarts""": get_cosine_with_hard_restarts_schedule_with_warmup,
    """polynomial""": get_polynomial_decay_schedule_with_warmup,
    """constant""": get_constant_schedule,
    """constant_w_warmup""": get_constant_schedule_with_warmup,
}

class   A							(						__lowercase    ):



      def __init__(						self:    Dict		,	_lowerCAmelCase:    List[str]=None		,	_lowerCAmelCase:    str=None		,	*_lowerCAmelCase:    Any		,	**_lowerCAmelCase:    Optional[int]							)    ->     Tuple:
         '''simple docstring'''

         super().__init__(*_lowerCAmelCase		,	**_lowerCAmelCase							)

         if config is None:
            assert isinstance(self.model		,	_lowerCAmelCase							), (
                "If no `config` is passed the model to be trained has to be of type `PreTrainedModel`, but is"
                F' {self.model.__class__}'
            )
            UpperCAmelCase_           =self.model.config
         else:
            UpperCAmelCase_           =config

         UpperCAmelCase_           =data_args
         UpperCAmelCase_           =self.config.tgt_vocab_size if isinstance(self.config		,	_lowerCAmelCase							) else self.config.vocab_size

         if self.args.label_smoothing != 0 or (self.data_args is not None and self.data_args.ignore_pad_token_for_loss):
            assert self.config.pad_token_id is not None, (
                "Make sure that `config.pad_token_id` is correcly defined when ignoring `pad_token` for loss"
                " calculation or doing label smoothing."
            )

         if self.config.pad_token_id is None and self.config.eos_token_id is not None:
            logger.warning(
                F'The `config.pad_token_id` is `None`. Using `config.eos_token_id` = {self.config.eos_token_id} for'
                " padding.."							)

         if self.args.label_smoothing == 0:
            UpperCAmelCase_           =torch.nn.CrossEntropyLoss(ignore_index=self.config.pad_token_id							)
         else:
            # dynamically import label_smoothed_nll_loss
            from utils import label_smoothed_nll_loss

            UpperCAmelCase_           =label_smoothed_nll_loss



      def 						lowerCAmelCase__     (						self:    str		,	_lowerCAmelCase:    int							)    ->     Optional[int]:
         '''simple docstring'''

         if self.optimizer is None:
            UpperCAmelCase_           =["bias", "LayerNorm.weight"]
            UpperCAmelCase_           =[
                {
                    "params": [p for n, p in self.model.named_parameters() if not any(nd in n for nd in no_decay							)],
                    "weight_decay": self.args.weight_decay,
                },
                {
                    "params": [p for n, p in self.model.named_parameters() if any(nd in n for nd in no_decay							)],
                    "weight_decay": 0.0,
                },
            ]
            UpperCAmelCase_           =Adafactor if self.args.adafactor else AdamW
            if self.args.adafactor:
               UpperCAmelCase_           =Adafactor
               UpperCAmelCase_           ={"scale_parameter": False, "relative_step": False}
            else:
               UpperCAmelCase_           =AdamW
               UpperCAmelCase_           ={
                   "betas": (self.args.adam_betaa, self.args.adam_betaa),
                   "eps": self.args.adam_epsilon,
               }
            UpperCAmelCase_           =self.args.learning_rate
            if self.sharded_ddp:
               UpperCAmelCase_           =OSS(
                   params=_lowerCAmelCase		,	optim=_lowerCAmelCase		,	**_lowerCAmelCase		,	)
            else:
               UpperCAmelCase_           =optimizer_cls(_lowerCAmelCase		,	**_lowerCAmelCase							)

         if self.lr_scheduler is None:
            UpperCAmelCase_           =self._get_lr_scheduler(_lowerCAmelCase							)
         else:  # ignoring --lr_scheduler
            logger.warning("scheduler is passed to `Seq2SeqTrainer`, `--lr_scheduler` arg is ignored."							)



      def 						lowerCAmelCase__     (						self:    Union[str, Any]		,	_lowerCAmelCase:    int							)    ->     Any:
         '''simple docstring'''

         UpperCAmelCase_           =arg_to_scheduler[self.args.lr_scheduler]
         if self.args.lr_scheduler == "constant":
            UpperCAmelCase_           =schedule_func(self.optimizer							)
         elif self.args.lr_scheduler == "constant_w_warmup":
            UpperCAmelCase_           =schedule_func(self.optimizer		,	num_warmup_steps=self.args.warmup_steps							)
         else:
            UpperCAmelCase_           =schedule_func(
                self.optimizer		,	num_warmup_steps=self.args.warmup_steps		,	num_training_steps=_lowerCAmelCase							)
         return scheduler



      def 						lowerCAmelCase__     (						self:    Optional[int]							)    ->     Optional[torch.utils.data.Sampler]:
         '''simple docstring'''

         if isinstance(self.train_dataset		,	torch.utils.data.IterableDataset							):
            return None
         elif is_torch_tpu_available():
            return get_tpu_sampler(self.train_dataset							)
         else:
            if self.args.sortish_sampler:
               self.train_dataset.make_sortish_sampler(
                   self.args.per_device_train_batch_size		,	distributed=(self.args.parallel_mode == ParallelMode.DISTRIBUTED)		,	)

            return (
                RandomSampler(self.train_dataset							)
                if self.args.local_rank == -1
                else DistributedSampler(self.train_dataset							)
            )



      def 						lowerCAmelCase__     (						self:    List[str]		,	_lowerCAmelCase:    Union[str, Any]		,	_lowerCAmelCase:    int		,	_lowerCAmelCase:    Optional[int]							)    ->     str:
         '''simple docstring'''

         if self.args.label_smoothing == 0:
            if self.data_args is not None and self.data_args.ignore_pad_token_for_loss:
               # force training to ignore pad token
               UpperCAmelCase_           =model(**_lowerCAmelCase		,	use_cache=_lowerCAmelCase							)[0]
               UpperCAmelCase_           =self.loss_fn(logits.view(-1		,	logits.shape[-1]							)		,	labels.view(-1							)							)
            else:
               # compute usual loss via models
               UpperCAmelCase_    ,							UpperCAmelCase_           =model(**_lowerCAmelCase		,	labels=_lowerCAmelCase		,	use_cache=_lowerCAmelCase							)[:2]
         else:
            # compute label smoothed loss
            UpperCAmelCase_           =model(**_lowerCAmelCase		,	use_cache=_lowerCAmelCase							)[0]
            UpperCAmelCase_           =torch.nn.functional.log_softmax(_lowerCAmelCase		,	dim=-1							)
            UpperCAmelCase_    ,							UpperCAmelCase_           =self.loss_fn(_lowerCAmelCase		,	_lowerCAmelCase		,	self.args.label_smoothing		,	ignore_index=self.config.pad_token_id							)
         return loss, logits



      def 						lowerCAmelCase__     (						self:    Optional[Any]		,	_lowerCAmelCase:    List[Any]		,	_lowerCAmelCase:    Any							)    ->     List[str]:
         '''simple docstring'''

         UpperCAmelCase_           =inputs.pop("labels"							)
         UpperCAmelCase_    ,							UpperCAmelCase_           =self._compute_loss(_lowerCAmelCase		,	_lowerCAmelCase		,	_lowerCAmelCase							)
         return loss



      def 						lowerCAmelCase__     (						self:    Union[str, Any]		,	_lowerCAmelCase:    nn.Module		,	_lowerCAmelCase:    Dict[str, Union[torch.Tensor, Any]]		,	_lowerCAmelCase:    bool		,	_lowerCAmelCase:    Optional[List[str]] = None		,	)    ->     Tuple[Optional[float], Optional[torch.Tensor], Optional[torch.Tensor]]:
         '''simple docstring'''

         UpperCAmelCase_           =self._prepare_inputs(_lowerCAmelCase							)

         UpperCAmelCase_           ={
             "max_length": self.data_args.val_max_target_length
             if self.data_args is not None
             else self.config.max_length,
             "num_beams": self.data_args.eval_beams if self.data_args is not None else self.config.num_beams,
         }

         if self.args.predict_with_generate and not self.args.prediction_loss_only:
            UpperCAmelCase_           =self.model.generate(
                inputs["input_ids"]		,	attention_mask=inputs["attention_mask"]		,	**_lowerCAmelCase		,	)
            # in case the batch is shorter than max length, the output should be padded
            if generated_tokens.shape[-1] < gen_kwargs["max_length"]:
               UpperCAmelCase_           =self._pad_tensors_to_max_len(_lowerCAmelCase		,	gen_kwargs["max_length"]							)

         UpperCAmelCase_           =inputs.pop("labels"							)
         with torch.no_grad():
            # compute loss on predict data
            UpperCAmelCase_    ,							UpperCAmelCase_           =self._compute_loss(_lowerCAmelCase		,	_lowerCAmelCase		,	_lowerCAmelCase							)

         UpperCAmelCase_           =loss.mean().detach()
         if self.args.prediction_loss_only:
            return (loss, None, None)

         UpperCAmelCase_           =generated_tokens if self.args.predict_with_generate else logits

         if labels.shape[-1] < gen_kwargs["max_length"]:
            UpperCAmelCase_           =self._pad_tensors_to_max_len(_lowerCAmelCase		,	gen_kwargs["max_length"]							)

         return (loss, logits, labels)






      def 						lowerCAmelCase__     (						self:    str		,	_lowerCAmelCase:    List[Any]		,	_lowerCAmelCase:    Optional[Any]							)    ->     Union[str, Any]:
         '''simple docstring'''

         UpperCAmelCase_           =self.config.pad_token_id if self.config.pad_token_id is not None else self.config.eos_token_id

         if pad_token_id is None:
            raise ValueError(
                "Make sure that either `config.pad_token_id` or `config.eos_token_id` is defined if tensor has to be"
                F' padded to `max_length`={max_length}'							)

         UpperCAmelCase_           =pad_token_id * torch.ones(
             (tensor.shape[0], max_length)		,	dtype=tensor.dtype		,	device=tensor.device							)
         UpperCAmelCase_           =tensor
         return padded_tensor






"""simple docstring"""




import json
import os
import re
import shutil
import tempfile
import unittest
from typing import Tuple

from transformers import AddedToken, BatchEncoding, PerceiverTokenizer
from transformers.utils import cached_property, is_tf_available, is_torch_available

from ...test_tokenization_common import TokenizerTesterMixin


if is_torch_available():
    a		:       str     						=       'pt'
elif is_tf_available():
    a		:       Dict     						=       'tf'
else:
    a		:       Optional[Any]     						=       'jax'







class     lowercase(_lowercase    ,						unittest.TestCase       ):
     __snake_case:	List[Any]						=     PerceiverTokenizer
     __snake_case:	Optional[Any]						=     False



     def 			lowercase__  (							self   )		->     Tuple:







         """simple docstring"""



         super().setUp()
         a__		=			PerceiverTokenizer()
         tokenizer.save_pretrained(self.tmpdirname   )



     @cached_property
     def 			lowercase__  (							self   )		->     Any:







         """simple docstring"""



         return PerceiverTokenizer.from_pretrained('deepmind/language-perceiver'   )



     def 			lowercase__  (							self     ,				**__SCREAMING_SNAKE_CASE   )		->     PerceiverTokenizer:







         """simple docstring"""



         return self.tokenizer_class.from_pretrained(self.tmpdirname     ,				**__SCREAMING_SNAKE_CASE   )



     def 			lowercase__  (							self     ,				__SCREAMING_SNAKE_CASE     ,				__SCREAMING_SNAKE_CASE=False     ,				__SCREAMING_SNAKE_CASE=2_0     ,				__SCREAMING_SNAKE_CASE=5   )		->     Tuple[str, list]:







         """simple docstring"""



         a__		=			[]
         for i in range(len(__SCREAMING_SNAKE_CASE   )   ):
             try:
                 a__		=			tokenizer.decode([i]     ,				clean_up_tokenization_spaces=__SCREAMING_SNAKE_CASE   )
             except UnicodeDecodeError:
                 pass
             toks.append((i, tok)   )

         a__		=			list(filter(lambda __SCREAMING_SNAKE_CASE   : re.match(R'^[ a-zA-Z]+$'     ,				t[1]   )     ,				__SCREAMING_SNAKE_CASE   )   )
         a__		=			list(filter(lambda __SCREAMING_SNAKE_CASE   : [t[0]] == tokenizer.encode(t[1]     ,				add_special_tokens=__SCREAMING_SNAKE_CASE   )     ,				__SCREAMING_SNAKE_CASE   )   )
         if max_length is not None and len(__SCREAMING_SNAKE_CASE   ) > max_length:
             a__		=			toks[:max_length]
         if min_length is not None and len(__SCREAMING_SNAKE_CASE   ) < min_length and len(__SCREAMING_SNAKE_CASE   ) > 0:
             while len(__SCREAMING_SNAKE_CASE   ) < min_length:
                 a__		=			toks + toks
        # toks_str = [t[1] for t in toks]
         a__		=			[t[0] for t in toks]

         # Ensure consistency
         a__		=			tokenizer.decode(__SCREAMING_SNAKE_CASE     ,				clean_up_tokenization_spaces=__SCREAMING_SNAKE_CASE   )
         if " " not in output_txt and len(__SCREAMING_SNAKE_CASE   ) > 1:
             a__		=			(
                 tokenizer.decode([toks_ids[0]]     ,				clean_up_tokenization_spaces=__SCREAMING_SNAKE_CASE   )
                 + ' '
                 + tokenizer.decode(toks_ids[1:]     ,				clean_up_tokenization_spaces=__SCREAMING_SNAKE_CASE   )
             )
         if with_prefix_space:
             a__		=			' ' + output_txt
         a__		=			tokenizer.encode(__SCREAMING_SNAKE_CASE     ,				add_special_tokens=__SCREAMING_SNAKE_CASE   )
         return output_txt, output_ids



     def 			lowercase__  (							self   )		->     Optional[Any]:







         """simple docstring"""



         a__		=			self.perceiver_tokenizer
         a__		=			'Unicode €.'
         a__		=			tokenizer(__SCREAMING_SNAKE_CASE   )
         a__		=			[4, 9_1, 1_1_6, 1_1_1, 1_0_5, 1_1_7, 1_0_6, 1_0_7, 3_8, 2_3_2, 1_3_6, 1_7_8, 5_2, 5]
         self.assertEqual(encoded['input_ids']     ,				__SCREAMING_SNAKE_CASE   )

         # decoding
         a__		=			tokenizer.decode(__SCREAMING_SNAKE_CASE   )
         self.assertEqual(__SCREAMING_SNAKE_CASE     ,				'[CLS]Unicode €.[SEP]'   )

         a__		=			tokenizer('e è é ê ë'   )
         a__		=			[4, 1_0_7, 3_8, 2_0_1, 1_7_4, 3_8, 2_0_1, 1_7_5, 3_8, 2_0_1, 1_7_6, 3_8, 2_0_1, 1_7_7, 5]
         self.assertEqual(encoded['input_ids']     ,				__SCREAMING_SNAKE_CASE   )
         # decoding
         a__		=			tokenizer.decode(__SCREAMING_SNAKE_CASE   )
         self.assertEqual(__SCREAMING_SNAKE_CASE     ,				'[CLS]e è é ê ë[SEP]'   )

         # encode/decode, but with `encode` instead of `__call__`
         self.assertEqual(tokenizer.decode(tokenizer.encode('e è é ê ë'   )   )     ,				'[CLS]e è é ê ë[SEP]'   )



     def 			lowercase__  (							self   )		->     List[str]:







         """simple docstring"""



         a__		=			self.perceiver_tokenizer
         a__		=			['A long paragraph for summarization.', 'Another paragraph for summarization.']
         # fmt: off
         a__		=			[4, 7_1, 3_8, 1_1_4, 1_1_7, 1_1_6, 1_0_9, 3_8, 1_1_8, 1_0_3, 1_2_0, 1_0_3, 1_0_9, 1_2_0, 1_0_3, 1_1_8, 1_1_0, 3_8, 1_0_8, 1_1_7, 1_2_0, 3_8, 1_2_1, 1_2_3, 1_1_5, 1_1_5, 1_0_3, 1_2_0, 1_1_1, 1_2_8, 1_0_3, 1_2_2, 1_1_1, 1_1_7, 1_1_6, 5_2, 5, 0]
         # fmt: on
         a__		=			tokenizer(__SCREAMING_SNAKE_CASE     ,				padding=__SCREAMING_SNAKE_CASE     ,				return_tensors=__SCREAMING_SNAKE_CASE   )
         self.assertIsInstance(__SCREAMING_SNAKE_CASE     ,				__SCREAMING_SNAKE_CASE   )

         if FRAMEWORK != "jax":
             a__		=			list(batch.input_ids.numpy()[0]   )
         else:
             a__		=			list(batch.input_ids.tolist()[0]   )

         self.assertListEqual(__SCREAMING_SNAKE_CASE     ,				__SCREAMING_SNAKE_CASE   )

         self.assertEqual((2, 3_8)     ,				batch.input_ids.shape   )
         self.assertEqual((2, 3_8)     ,				batch.attention_mask.shape   )



     def 			lowercase__  (							self   )		->     Tuple:







         """simple docstring"""



         a__		=			self.perceiver_tokenizer
         a__		=			['A long paragraph for summarization.', 'Another paragraph for summarization.']
         a__		=			tokenizer(__SCREAMING_SNAKE_CASE     ,				padding=__SCREAMING_SNAKE_CASE     ,				return_tensors=__SCREAMING_SNAKE_CASE   )
         # check if input_ids are returned and no decoder_input_ids
         self.assertIn('input_ids'     ,				__SCREAMING_SNAKE_CASE   )
         self.assertIn('attention_mask'     ,				__SCREAMING_SNAKE_CASE   )
         self.assertNotIn('decoder_input_ids'     ,				__SCREAMING_SNAKE_CASE   )
         self.assertNotIn('decoder_attention_mask'     ,				__SCREAMING_SNAKE_CASE   )



     def 			lowercase__  (							self   )		->     Union[str, Any]:







         """simple docstring"""



         a__		=			self.perceiver_tokenizer
         a__		=			[
             'Summary of the text.',
             'Another summary.',
         ]
         a__		=			tokenizer(
             text_target=__SCREAMING_SNAKE_CASE     ,				max_length=3_2     ,				padding='max_length'     ,				truncation=__SCREAMING_SNAKE_CASE     ,				return_tensors=__SCREAMING_SNAKE_CASE   )
         self.assertEqual(3_2     ,				targets['input_ids'].shape[1]   )



     def 			lowercase__  (							self   )		->     Optional[int]:







         """simple docstring"""



         a__		=			self.get_tokenizers()
         for tokenizer in tokenizers:
             with self.subTest(f'{tokenizer.__class__.__name__}'   ):
                 self.assertNotEqual(tokenizer.model_max_length     ,				4_2   )

        # Now let's start the test
         a__		=			self.get_tokenizers()
         for tokenizer in tokenizers:
             with self.subTest(f'{tokenizer.__class__.__name__}'   ):
                 # Isolate this from the other tests because we save additional tokens/etc
                 a__		=			tempfile.mkdtemp()

                 a__		=			' He is very happy, UNwant\u00E9d,running'
                 a__		=			tokenizer.encode(__SCREAMING_SNAKE_CASE     ,				add_special_tokens=__SCREAMING_SNAKE_CASE   )
                 tokenizer.save_pretrained(__SCREAMING_SNAKE_CASE   )

                 a__		=			tokenizer.__class__.from_pretrained(__SCREAMING_SNAKE_CASE   )
                 a__		=			after_tokenizer.encode(__SCREAMING_SNAKE_CASE     ,				add_special_tokens=__SCREAMING_SNAKE_CASE   )
                 self.assertListEqual(__SCREAMING_SNAKE_CASE     ,				__SCREAMING_SNAKE_CASE   )

                 shutil.rmtree(__SCREAMING_SNAKE_CASE   )

         a__		=			self.get_tokenizers(model_max_length=4_2   )
         for tokenizer in tokenizers:
             with self.subTest(f'{tokenizer.__class__.__name__}'   ):
                 # Isolate this from the other tests because we save additional tokens/etc
                 a__		=			tempfile.mkdtemp()

                 a__		=			' He is very happy, UNwant\u00E9d,running'
                 tokenizer.add_tokens(['bim', 'bambam']   )
                 a__		=			tokenizer.additional_special_tokens
                 additional_special_tokens.append('new_additional_special_token'   )
                 tokenizer.add_special_tokens({'additional_special_tokens': additional_special_tokens}   )
                 a__		=			tokenizer.encode(__SCREAMING_SNAKE_CASE     ,				add_special_tokens=__SCREAMING_SNAKE_CASE   )
                 tokenizer.save_pretrained(__SCREAMING_SNAKE_CASE   )

                 a__		=			tokenizer.__class__.from_pretrained(__SCREAMING_SNAKE_CASE   )
                 a__		=			after_tokenizer.encode(__SCREAMING_SNAKE_CASE     ,				add_special_tokens=__SCREAMING_SNAKE_CASE   )
                 self.assertListEqual(__SCREAMING_SNAKE_CASE     ,				__SCREAMING_SNAKE_CASE   )
                 self.assertIn('new_additional_special_token'     ,				after_tokenizer.additional_special_tokens   )
                 self.assertEqual(after_tokenizer.model_max_length     ,				4_2   )

                 a__		=			tokenizer.__class__.from_pretrained(__SCREAMING_SNAKE_CASE     ,				model_max_length=4_3   )
                 self.assertEqual(tokenizer.model_max_length     ,				4_3   )

                 shutil.rmtree(__SCREAMING_SNAKE_CASE   )



     def 			lowercase__  (							self   )		->     List[Any]:







         """simple docstring"""



         a__		=			[]
         if self.test_slow_tokenizer:
             tokenizer_list.append((self.tokenizer_class, self.get_tokenizer())   )

         if self.test_rust_tokenizer:
             tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer())   )

         for tokenizer_class, tokenizer_utils in tokenizer_list:
             with tempfile.TemporaryDirectory() as tmp_dir:
                 tokenizer_utils.save_pretrained(__SCREAMING_SNAKE_CASE   )

                 with open(os.path.join(__SCREAMING_SNAKE_CASE     ,				'special_tokens_map.json'   )     ,				encoding='utf-8'   ) as json_file:
                     a__		=			json.load(__SCREAMING_SNAKE_CASE   )

                 with open(os.path.join(__SCREAMING_SNAKE_CASE     ,				'tokenizer_config.json'   )     ,				encoding='utf-8'   ) as json_file:
                     a__		=			json.load(__SCREAMING_SNAKE_CASE   )

                 a__		=			[f'<extra_id_{i}>' for i in range(1_2_5   )]

                 a__		=			added_tokens_extra_ids + [
                     'an_additional_special_token'
                 ]
                 a__		=			added_tokens_extra_ids + [
                     'an_additional_special_token'
                 ]

                 with open(os.path.join(__SCREAMING_SNAKE_CASE     ,				'special_tokens_map.json'   )     ,				'w'     ,				encoding='utf-8'   ) as outfile:
                     json.dump(__SCREAMING_SNAKE_CASE     ,				__SCREAMING_SNAKE_CASE   )
                 with open(os.path.join(__SCREAMING_SNAKE_CASE     ,				'tokenizer_config.json'   )     ,				'w'     ,				encoding='utf-8'   ) as outfile:
                     json.dump(__SCREAMING_SNAKE_CASE     ,				__SCREAMING_SNAKE_CASE   )

                 # the following checks allow us to verify that our test works as expected, i.e. that the tokenizer takes
                 # into account the new value of additional_special_tokens given in the "tokenizer_config.json" and
                 # "special_tokens_map.json" files
                 a__		=			tokenizer_class.from_pretrained(
                     __SCREAMING_SNAKE_CASE     ,				)
                 self.assertIn(
                     'an_additional_special_token'     ,				tokenizer_without_change_in_init.additional_special_tokens   )
                 self.assertEqual(
                     ['an_additional_special_token']     ,				tokenizer_without_change_in_init.convert_ids_to_tokens(
                         tokenizer_without_change_in_init.convert_tokens_to_ids(['an_additional_special_token']   )   )     ,				)

                 # Now we test that we can change the value of additional_special_tokens in the from_pretrained
                 a__		=			added_tokens_extra_ids + [AddedToken('a_new_additional_special_token'     ,				lstrip=__SCREAMING_SNAKE_CASE   )]
                 a__		=			tokenizer_class.from_pretrained(
                     __SCREAMING_SNAKE_CASE     ,				additional_special_tokens=__SCREAMING_SNAKE_CASE     ,				)

                 self.assertIn('a_new_additional_special_token'     ,				tokenizer.additional_special_tokens   )
                 self.assertEqual(
                     ['a_new_additional_special_token']     ,				tokenizer.convert_ids_to_tokens(
                         tokenizer.convert_tokens_to_ids(['a_new_additional_special_token']   )   )     ,				)



     def 			lowercase__  (							self   )		->     Union[str, Any]:







         """simple docstring"""



         a__		=			self.perceiver_tokenizer
         self.assertEqual(tokenizer.decode([1_7_8]   )     ,				'�'   )



     def 			lowercase__  (							self   )		->     str:







         """simple docstring"""



         pass



     def 			lowercase__  (							self   )		->     List[str]:







         """simple docstring"""



         pass



     def 			lowercase__  (							self   )		->     Dict:







         """simple docstring"""



         pass



     def 			lowercase__  (							self   )		->     List[str]:







         """simple docstring"""



         pass




     def 			lowercase__  (							self   )		->     Union[str, Any]:







         """simple docstring"""



         a__		=			self.get_tokenizers(fast=__SCREAMING_SNAKE_CASE     ,				do_lower_case=__SCREAMING_SNAKE_CASE   )
         for tokenizer in tokenizers:
             with self.subTest(f'{tokenizer.__class__.__name__}'   ):
                 a__		=			['[CLS]', 't', 'h', 'i', 's', ' ', 'i', 's', ' ', 'a', ' ', 't', 'e', 's', 't', '[SEP]']
                 a__		=			tokenizer.convert_tokens_to_string(__SCREAMING_SNAKE_CASE   )

                 self.assertIsInstance(__SCREAMING_SNAKE_CASE     ,				__SCREAMING_SNAKE_CASE   )


