



"""simple docstring"""






import unittest

from transformers import AutoTokenizer, FalconConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
     import torch

     from transformers import (
         FalconForCausalLM,
         FalconForQuestionAnswering,
         FalconForSequenceClassification,
         FalconForTokenClassification,
         FalconModel,
     )



class     UpperCAmelCase    :



   def __init__(       self    :	int							,       __lowerCamelCase    :	Optional[Any]							,       __lowerCamelCase    :	Optional[Any]=3							,       __lowerCamelCase    :	Optional[int]=7							,       __lowerCamelCase    :	Optional[int]=True							,       __lowerCamelCase    :	Optional[int]=True							,       __lowerCamelCase    :	Union[str, Any]=False							,       __lowerCamelCase    :	int=True							,       __lowerCamelCase    :	Any=9_9							,       __lowerCamelCase    :	Tuple=3_2							,       __lowerCamelCase    :	str=5							,       __lowerCamelCase    :	List[Any]=4							,       __lowerCamelCase    :	List[str]=3_7							,       __lowerCamelCase    :	List[Any]="gelu"							,       __lowerCamelCase    :	Tuple=0.1							,       __lowerCamelCase    :	str=0.1							,       __lowerCamelCase    :	int=5_1_2							,       __lowerCamelCase    :	Dict=1_6							,       __lowerCamelCase    :	str=2							,       __lowerCamelCase    :	Optional[int]=0.0_2							,       __lowerCamelCase    :	Union[str, Any]=3							,       __lowerCamelCase    :	Dict=4							,       __lowerCamelCase    :	Union[str, Any]=None							,       ):



     """simple docstring"""
     _snake_case   	=      parent
     _snake_case   	=      batch_size
     _snake_case   	=      seq_length
     _snake_case   	=      is_training
     _snake_case   	=      use_input_mask
     _snake_case   	=      use_token_type_ids
     _snake_case   	=      use_labels
     _snake_case   	=      vocab_size
     _snake_case   	=      hidden_size
     _snake_case   	=      num_hidden_layers
     _snake_case   	=      num_attention_heads
     _snake_case   	=      intermediate_size
     _snake_case   	=      hidden_act
     _snake_case   	=      hidden_dropout_prob
     _snake_case   	=      attention_probs_dropout_prob
     _snake_case   	=      max_position_embeddings
     _snake_case   	=      type_vocab_size
     _snake_case   	=      type_sequence_label_size
     _snake_case   	=      initializer_range
     _snake_case   	=      num_labels
     _snake_case   	=      num_choices
     _snake_case   	=      scope



   def    __UpperCAmelCase    (       self    :	List[Any]				):



     """simple docstring"""
     _snake_case   	=      ids_tensor([self.batch_size, self.seq_length]							,       self.vocab_size				)

     _snake_case   	=      None
     if self.use_input_mask:
       _snake_case   	=      random_attention_mask([self.batch_size, self.seq_length]				)

     _snake_case   	=      None

     _snake_case   	=      None
     _snake_case   	=      None
     _snake_case   	=      None
     if self.use_labels:
       _snake_case   	=      ids_tensor([self.batch_size]							,       self.type_sequence_label_size				)
       _snake_case   	=      ids_tensor([self.batch_size, self.seq_length]							,       self.num_labels				)
       _snake_case   	=      ids_tensor([self.batch_size]							,       self.num_choices				)

     _snake_case   	=      self.get_config()

     return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels



   def    __UpperCAmelCase    (       self    :	List[Any]				):



     """simple docstring"""
     return FalconConfig(
         vocab_size=self.vocab_size							,       hidden_size=self.hidden_size							,       num_hidden_layers=self.num_hidden_layers							,       num_attention_heads=self.num_attention_heads							,       intermediate_size=self.intermediate_size							,       hidden_act=self.hidden_act							,       hidden_dropout_prob=self.hidden_dropout_prob							,       attention_probs_dropout_prob=self.attention_probs_dropout_prob							,       max_position_embeddings=self.max_position_embeddings							,       type_vocab_size=self.type_vocab_size							,       is_decoder=__lowerCamelCase							,       initializer_range=self.initializer_range							,       pad_token_id=1							,       new_decoder_architecture=__lowerCamelCase							,       )



   def    __UpperCAmelCase    (       self    :	List[str]							,       __lowerCamelCase    :	Tuple							,       __lowerCamelCase    :	Dict							,       __lowerCamelCase    :	int							,       __lowerCamelCase    :	Union[str, Any]							,       __lowerCamelCase    :	Optional[Any]							,       __lowerCamelCase    :	List[Any]							,       __lowerCamelCase    :	List[str]				):



     """simple docstring"""
     _snake_case   	=      FalconModel(config=__lowerCamelCase				)
     model.to(__lowerCamelCase				)
     model.eval()
     _snake_case   	=      model(__lowerCamelCase							,       attention_mask=__lowerCamelCase				)
     _snake_case   	=      model(__lowerCamelCase				)
     self.parent.assertEqual(result.last_hidden_state.shape							,       (self.batch_size, self.seq_length, self.hidden_size)				)



   def    __UpperCAmelCase    (       self    :	Dict							,       __lowerCamelCase    :	str							,       __lowerCamelCase    :	str							,       __lowerCamelCase    :	Any							,       __lowerCamelCase    :	Tuple							,       __lowerCamelCase    :	Any							,       __lowerCamelCase    :	List[Any]							,       __lowerCamelCase    :	Union[str, Any]							,       __lowerCamelCase    :	int							,       __lowerCamelCase    :	str							,       ):



     """simple docstring"""
     _snake_case   	=      True
     _snake_case   	=      FalconModel(__lowerCamelCase				)
     model.to(__lowerCamelCase				)
     model.eval()
     _snake_case   	=      model(
         __lowerCamelCase							,       attention_mask=__lowerCamelCase							,       encoder_hidden_states=__lowerCamelCase							,       encoder_attention_mask=__lowerCamelCase							,       )
     _snake_case   	=      model(
         __lowerCamelCase							,       attention_mask=__lowerCamelCase							,       encoder_hidden_states=__lowerCamelCase							,       )
     _snake_case   	=      model(__lowerCamelCase							,       attention_mask=__lowerCamelCase				)
     self.parent.assertEqual(result.last_hidden_state.shape							,       (self.batch_size, self.seq_length, self.hidden_size)				)



   def    __UpperCAmelCase    (       self    :	List[Any]							,       __lowerCamelCase    :	str							,       __lowerCamelCase    :	List[str]							,       __lowerCamelCase    :	Tuple							,       __lowerCamelCase    :	int							,       __lowerCamelCase    :	int							,       __lowerCamelCase    :	List[str]							,       __lowerCamelCase    :	Optional[int]							,       __lowerCamelCase    :	Tuple							,       __lowerCamelCase    :	str							,       ):



     """simple docstring"""
     _snake_case   	=      FalconForCausalLM(config=__lowerCamelCase				)
     model.to(__lowerCamelCase				)
     model.eval()
     _snake_case   	=      model(__lowerCamelCase							,       attention_mask=__lowerCamelCase							,       labels=__lowerCamelCase				)
     self.parent.assertEqual(result.logits.shape							,       (self.batch_size, self.seq_length, self.vocab_size)				)



   def    __UpperCAmelCase    (       self    :	Dict							,       __lowerCamelCase    :	Dict							,       __lowerCamelCase    :	List[str]							,       __lowerCamelCase    :	List[Any]							,       __lowerCamelCase    :	str							,       __lowerCamelCase    :	Dict							,       __lowerCamelCase    :	Tuple							,       __lowerCamelCase    :	List[str]							,       __lowerCamelCase    :	int							,       __lowerCamelCase    :	str							,       ):



     """simple docstring"""
     _snake_case   	=      True
     _snake_case   	=      True
     _snake_case   	=      FalconForCausalLM(config=__lowerCamelCase				)
     model.to(__lowerCamelCase				)
     model.eval()

     # first forward pass
     _snake_case   	=      model(
         __lowerCamelCase							,       attention_mask=__lowerCamelCase							,       encoder_hidden_states=__lowerCamelCase							,       encoder_attention_mask=__lowerCamelCase							,       use_cache=__lowerCamelCase							,       )
     _snake_case   	=      outputs.past_key_values

     # create hypothetical multiple next token and extent to next_input_ids
     _snake_case   	=      ids_tensor((self.batch_size, 3)							,       config.vocab_size				)
     _snake_case   	=      ids_tensor((self.batch_size, 3)							,       vocab_size=2				)

     # append to next input_ids and
     _snake_case   	=      torch.cat([input_ids, next_tokens]							,       dim=-1				)
     _snake_case   	=      torch.cat([input_mask, next_mask]							,       dim=-1				)

     _snake_case   	=      model(
         __lowerCamelCase							,       attention_mask=__lowerCamelCase							,       encoder_hidden_states=__lowerCamelCase							,       encoder_attention_mask=__lowerCamelCase							,       output_hidden_states=__lowerCamelCase							,       )['''hidden_states'''][0]
     _snake_case   	=      model(
         __lowerCamelCase							,       attention_mask=__lowerCamelCase							,       encoder_hidden_states=__lowerCamelCase							,       encoder_attention_mask=__lowerCamelCase							,       past_key_values=__lowerCamelCase							,       output_hidden_states=__lowerCamelCase							,       )['''hidden_states'''][0]

     # select random slice
     _snake_case   	=      ids_tensor((1,)							,       output_from_past.shape[-1]				).item()
     _snake_case   	=      output_from_no_past[:, -3:, random_slice_idx].detach()
     _snake_case   	=      output_from_past[:, :, random_slice_idx].detach()

     self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1]				)

     # test that outputs are equal for slice
     self.parent.assertTrue(torch.allclose(__lowerCamelCase							,       __lowerCamelCase							,       atol=1E-3				)				)





   def    __UpperCAmelCase    (       self    :	Union[str, Any]				):



     """simple docstring"""
     _snake_case   	=      self.prepare_config_and_inputs()
     (
         (
         _snake_case
     )	,     (
         _snake_case
     )	,     (
         _snake_case
     )	,     (
         _snake_case
     )	,     (
         _snake_case
     )	,     (
         _snake_case
     )	,     (
         _snake_case
     )	,     
     )   	=      config_and_inputs
     _snake_case   	=      {'''input_ids''': input_ids, '''attention_mask''': input_mask}
     return config, inputs_dict



@require_torch
class     UpperCAmelCase    (	__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,unittest.TestCase  ):
   A__		:	Optional[Any]           =  (
       (
           FalconModel,
           FalconForCausalLM,
           FalconForSequenceClassification,
           FalconForTokenClassification,
           FalconForQuestionAnswering,
       )
       if is_torch_available()
       else ()
   )
   A__		:	Tuple           =  (FalconForCausalLM,) if is_torch_available() else ()
   A__		:	Optional[Any]           =  (
       {
           '''feature-extraction''': FalconModel,
           '''text-classification''': FalconForSequenceClassification,
           '''text-generation''': FalconForCausalLM,
           '''question-answering''': FalconForQuestionAnswering,
           '''token-classification''': FalconForTokenClassification,
           '''zero-shot''': FalconForSequenceClassification,
       }
       if is_torch_available()
       else {}
   )
   A__		:	List[Any]           =  False
   A__		:	List[Any]           =  False



   def    __UpperCAmelCase    (       self    :	Optional[int]				):



     """simple docstring"""
     _snake_case   	=      FalconModelTester(self				)
     _snake_case   	=      ConfigTester(self							,       config_class=__lowerCamelCase							,       hidden_size=3_7				)



   def    __UpperCAmelCase    (       self    :	Any				):



     """simple docstring"""
     self.config_tester.run_common_tests()



   def    __UpperCAmelCase    (       self    :	Any				):



     """simple docstring"""
     _snake_case   	=      self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_model(*__lowerCamelCase				)



   def    __UpperCAmelCase    (       self    :	Optional[int]				):



     """simple docstring"""
     _snake_case	,     *_snake_case   	=      self.model_tester.prepare_config_and_inputs()
     for alibi in [True, False]:
       _snake_case   	=      alibi
       self.model_tester.create_and_check_model(__lowerCamelCase							,       *__lowerCamelCase				)



   def    __UpperCAmelCase    (       self    :	List[Any]				):



     """simple docstring"""
     _snake_case	,     _snake_case   	=      self.model_tester.prepare_config_and_inputs_for_common()
     _snake_case   	=      3
     _snake_case   	=      input_dict['''input_ids''']
     _snake_case   	=      input_ids.ne(1				).to(__lowerCamelCase				)
     _snake_case   	=      ids_tensor([self.model_tester.batch_size]							,       self.model_tester.type_sequence_label_size				)
     _snake_case   	=      FalconForSequenceClassification(__lowerCamelCase				)
     model.to(__lowerCamelCase				)
     model.eval()
     _snake_case   	=      model(__lowerCamelCase							,       attention_mask=__lowerCamelCase							,       labels=__lowerCamelCase				)
     self.assertEqual(result.logits.shape							,       (self.model_tester.batch_size, self.model_tester.num_labels)				)



   def    __UpperCAmelCase    (       self    :	Optional[Any]				):



     """simple docstring"""
     _snake_case	,     _snake_case   	=      self.model_tester.prepare_config_and_inputs_for_common()
     _snake_case   	=      3
     _snake_case   	=      '''single_label_classification'''
     _snake_case   	=      input_dict['''input_ids''']
     _snake_case   	=      input_ids.ne(1				).to(__lowerCamelCase				)
     _snake_case   	=      ids_tensor([self.model_tester.batch_size]							,       self.model_tester.type_sequence_label_size				)
     _snake_case   	=      FalconForSequenceClassification(__lowerCamelCase				)
     model.to(__lowerCamelCase				)
     model.eval()
     _snake_case   	=      model(__lowerCamelCase							,       attention_mask=__lowerCamelCase							,       labels=__lowerCamelCase				)
     self.assertEqual(result.logits.shape							,       (self.model_tester.batch_size, self.model_tester.num_labels)				)



   def    __UpperCAmelCase    (       self    :	List[str]				):



     """simple docstring"""
     _snake_case	,     _snake_case   	=      self.model_tester.prepare_config_and_inputs_for_common()
     _snake_case   	=      input_dict['''input_ids''']
     _snake_case   	=      FalconForCausalLM(__lowerCamelCase				)
     model.to(__lowerCamelCase				)
     model.eval()
     _snake_case   	=      model(__lowerCamelCase							,       use_cache=__lowerCamelCase				)
     _snake_case   	=      input_ids.shape[0]
     _snake_case   	=      model._convert_to_rw_cache(result.past_key_values				)
     _snake_case   	=      model._convert_cache_to_standard_format(__lowerCamelCase							,       __lowerCamelCase				)
     for layer in range(len(__lowerCamelCase				)				):
       for tensor_idx in range(2				):
         self.assertTrue(rw_cache[layer][tensor_idx].ndim == 3				)
         self.assertTrue(result.past_key_values[layer][tensor_idx].ndim == 4				)
         self.assertTrue(
             torch.all(result.past_key_values[layer][tensor_idx] == standard_cache[layer][tensor_idx]				)				)



   def    __UpperCAmelCase    (       self    :	Optional[Any]				):



     """simple docstring"""
     _snake_case	,     _snake_case   	=      self.model_tester.prepare_config_and_inputs_for_common()
     _snake_case   	=      3
     _snake_case   	=      '''multi_label_classification'''
     _snake_case   	=      input_dict['''input_ids''']
     _snake_case   	=      input_ids.ne(1				).to(__lowerCamelCase				)
     _snake_case   	=      ids_tensor(
         [self.model_tester.batch_size, config.num_labels]							,       self.model_tester.type_sequence_label_size				).to(torch.float				)
     _snake_case   	=      FalconForSequenceClassification(__lowerCamelCase				)
     model.to(__lowerCamelCase				)
     model.eval()
     _snake_case   	=      model(__lowerCamelCase							,       attention_mask=__lowerCamelCase							,       labels=__lowerCamelCase				)
     self.assertEqual(result.logits.shape							,       (self.model_tester.batch_size, self.model_tester.num_labels)				)





   def    __UpperCAmelCase    (       self    :	Union[str, Any]				):



     """simple docstring"""
     # Falcon can have different numbers of KV-heads than the number of query heads, so we need
     # to override this test to use the right head counts.
     for model_class in self.all_generative_model_classes:
       _snake_case	,     _snake_case   	=      self.model_tester.prepare_config_and_inputs_for_common()

       # If it doesn't support cache, pass the test
       if not hasattr(__lowerCamelCase							,       '''use_cache'''				):
         return

       _snake_case   	=      model_class(__lowerCamelCase				).to(__lowerCamelCase				)
       if "use_cache" not in inputs:
         _snake_case   	=      True
       _snake_case   	=      model(**__lowerCamelCase				)

       # If "past_key_values" is not returned, pass the test (e.g. RWKV uses a different cache name and format)
       if "past_key_values" not in outputs:
         return

       _snake_case   	=      (
           getattr(__lowerCamelCase							,       '''decoder_layers'''							,       __lowerCamelCase				)
           or getattr(__lowerCamelCase							,       '''num_decoder_layers'''							,       __lowerCamelCase				)
           or config.num_hidden_layers
       )
       _snake_case   	=      getattr(__lowerCamelCase							,       '''num_kv_heads'''							,       config.num_attention_heads				)
       _snake_case   	=      getattr(__lowerCamelCase							,       '''d_model'''							,       config.hidden_size				)
       _snake_case   	=      embed_dim // num_attention_heads

       _snake_case   	=      outputs['''past_key_values''']
       self.assertEqual(len(__lowerCamelCase				)							,       __lowerCamelCase				)

       _snake_case	,     _snake_case   	=      inputs['''input_ids'''].shape
       for i in range(__lowerCamelCase				):
         if config.new_decoder_architecture:
           _snake_case   	=      config.num_attention_heads
         elif config.multi_query:
           _snake_case   	=      1
         self.assertEqual(len(past_kv[0]				)							,       2				)  # K V for the decoder = 2
         self.assertEqual(
             past_kv[i][0].shape							,       (batch_size, num_attention_heads, seq_length, per_head_embed_dim)				)
         self.assertEqual(
             past_kv[i][1].shape							,       (batch_size, num_attention_heads, seq_length, per_head_embed_dim)				)



@require_torch
class     UpperCAmelCase    (	unittest.TestCase  ):



   @slow
   def    __UpperCAmelCase    (       self    :	int				):



     """simple docstring"""
     _snake_case   	=      AutoTokenizer.from_pretrained('''Rocketknight1/falcon-rw-1b'''				)
     _snake_case   	=      FalconForCausalLM.from_pretrained('''Rocketknight1/falcon-rw-1b'''				)
     model.eval()
     model.to(__lowerCamelCase				)
     _snake_case   	=      tokenizer('''My favorite food is'''							,       return_tensors='''pt'''				).to(__lowerCamelCase				)

     _snake_case   	=      (
         '''My favorite food is pizza. I love it so much that I have a pizza party every year for my birthday.'''
     )

     _snake_case   	=      model.generate(**__lowerCamelCase							,       do_sample=__lowerCamelCase							,       max_new_tokens=1_9				)
     _snake_case   	=      tokenizer.batch_decode(__lowerCamelCase				)[0]

     self.assertEqual(__lowerCamelCase							,       __lowerCamelCase				)



   @slow
   def    __UpperCAmelCase    (       self    :	Union[str, Any]				):



     """simple docstring"""
     # The big models are way too big for the CI, so we use tiny random models that resemble their
     # architectures but with much smaller and fewer layers
     for repo in ["Rocketknight1/tiny-random-falcon-7b", "Rocketknight1/tiny-random-falcon-40b"]:
       _snake_case   	=      AutoTokenizer.from_pretrained(__lowerCamelCase				)
       _snake_case   	=      FalconForCausalLM.from_pretrained(__lowerCamelCase				)
       model.eval()
       model.to(__lowerCamelCase				)
       _snake_case   	=      tokenizer('''My favorite food is'''							,       return_tensors='''pt'''				).to(__lowerCamelCase				)

       # We just test that these run without errors - the models are randomly initialized
       # and so the actual text outputs will be garbage
       model.generate(**__lowerCamelCase							,       do_sample=__lowerCamelCase							,       max_new_tokens=4				)
       model.generate(**__lowerCamelCase							,       do_sample=__lowerCamelCase							,       max_new_tokens=4				)
       model.generate(**__lowerCamelCase							,       num_beams=2							,       max_new_tokens=4				)





   @slow
   def    __UpperCAmelCase    (       self    :	Union[str, Any]				):



     """simple docstring"""
     # The big models are way too big for the CI, so we use tiny random models that resemble their
     # architectures but with much smaller and fewer layers
     with torch.no_grad():
       for repo in [
           "Rocketknight1/falcon-rw-1b",
           "Rocketknight1/tiny-random-falcon-7b",
           "Rocketknight1/tiny-random-falcon-40b",
       ]:
         _snake_case   	=      AutoTokenizer.from_pretrained(__lowerCamelCase				)
         _snake_case   	=      FalconForCausalLM.from_pretrained(__lowerCamelCase				)
         model.eval()
         model.to(device=__lowerCamelCase				)
         _snake_case   	=      tokenizer('''My favorite food is'''							,       return_tensors='''pt'''				).to(__lowerCamelCase				)

         # Test results are the same with and without cache
         _snake_case   	=      model.generate(**__lowerCamelCase							,       do_sample=__lowerCamelCase							,       max_new_tokens=2_0							,       use_cache=__lowerCamelCase				)
         _snake_case   	=      model.generate(**__lowerCamelCase							,       do_sample=__lowerCamelCase							,       max_new_tokens=2_0							,       use_cache=__lowerCamelCase				)
         self.assertTrue((outputs_cache - outputs_no_cache).sum().item() == 0				)










"""simple docstring"""






import json
import os
import re
import shutil
import tempfile
import unittest
from typing import Tuple

from transformers import AddedToken, BatchEncoding, PerceiverTokenizer
from transformers.utils import cached_property, is_tf_available, is_torch_available

from ...test_tokenization_common import TokenizerTesterMixin


if is_torch_available():
     snake_case   					=				'''pt'''
elif is_tf_available():
     snake_case   					=				'''tf'''
else:
     snake_case   					=				'''jax'''



class     UpperCAmelCase    (	__SCREAMING_SNAKE_CASE,unittest.TestCase  ):
   A__		:	Union[str, Any]           =  PerceiverTokenizer
   A__		:	Optional[int]           =  False



   def    __UpperCAmelCase    (       self    :	List[Any]				):



     """simple docstring"""
     super().setUp()
     _snake_case   	=      PerceiverTokenizer()
     tokenizer.save_pretrained(self.tmpdirname				)



   @cached_property
   def    __UpperCAmelCase    (       self    :	int				):



     """simple docstring"""
     return PerceiverTokenizer.from_pretrained('''deepmind/language-perceiver'''				)



   def    __UpperCAmelCase    (       self    :	Union[str, Any]							,       **__lowerCamelCase    :	Union[str, Any]				):



     """simple docstring"""
     return self.tokenizer_class.from_pretrained(self.tmpdirname							,       **__lowerCamelCase				)



   def    __UpperCAmelCase    (       self    :	Dict							,       __lowerCamelCase    :	List[Any]							,       __lowerCamelCase    :	List[str]=False							,       __lowerCamelCase    :	int=2_0							,       __lowerCamelCase    :	List[Any]=5				):



     """simple docstring"""
     # XXX The default common tokenizer tests assume that every ID is decodable on its own.
     # This assumption is invalid for Perceiver because single bytes might not be
     # valid utf-8 (byte 128 for instance).
     # Here we're overriding the smallest possible method to provide
     # a clean sequence without making the same assumption.

     _snake_case   	=      []
     for i in range(len(__lowerCamelCase				)				):
       try:
         _snake_case   	=      tokenizer.decode([i]							,       clean_up_tokenization_spaces=__lowerCamelCase				)
       except UnicodeDecodeError:
         pass
       toks.append((i, tok)				)

     _snake_case   	=      list(filter(lambda __lowerCamelCase				: re.match(R'''^[ a-zA-Z]+$'''							,       t[1]				)							,       __lowerCamelCase				)				)
     _snake_case   	=      list(filter(lambda __lowerCamelCase				: [t[0]] == tokenizer.encode(t[1]							,       add_special_tokens=__lowerCamelCase				)							,       __lowerCamelCase				)				)
     if max_length is not None and len(__lowerCamelCase				) > max_length:
       _snake_case   	=      toks[:max_length]
     if min_length is not None and len(__lowerCamelCase				) < min_length and len(__lowerCamelCase				) > 0:
       while len(__lowerCamelCase				) < min_length:
         _snake_case   	=      toks + toks
        # toks_str = [t[1] for t in toks]
     _snake_case   	=      [t[0] for t in toks]

     # Ensure consistency
     _snake_case   	=      tokenizer.decode(__lowerCamelCase							,       clean_up_tokenization_spaces=__lowerCamelCase				)
     if " " not in output_txt and len(__lowerCamelCase				) > 1:
       _snake_case   	=      (
           tokenizer.decode([toks_ids[0]]							,       clean_up_tokenization_spaces=__lowerCamelCase				)
           + ''' '''
           + tokenizer.decode(toks_ids[1:]							,       clean_up_tokenization_spaces=__lowerCamelCase				)
       )
     if with_prefix_space:
       _snake_case   	=      ''' ''' + output_txt
     _snake_case   	=      tokenizer.encode(__lowerCamelCase							,       add_special_tokens=__lowerCamelCase				)
     return output_txt, output_ids



   def    __UpperCAmelCase    (       self    :	List[str]				):



     """simple docstring"""
     _snake_case   	=      self.perceiver_tokenizer
     _snake_case   	=      '''Unicode €.'''
     _snake_case   	=      tokenizer(__lowerCamelCase				)
     _snake_case   	=      [4, 9_1, 1_1_6, 1_1_1, 1_0_5, 1_1_7, 1_0_6, 1_0_7, 3_8, 2_3_2, 1_3_6, 1_7_8, 5_2, 5]
     self.assertEqual(encoded['''input_ids''']							,       __lowerCamelCase				)

     # decoding
     _snake_case   	=      tokenizer.decode(__lowerCamelCase				)
     self.assertEqual(__lowerCamelCase							,       '''[CLS]Unicode €.[SEP]'''				)

     _snake_case   	=      tokenizer('''e è é ê ë'''				)
     _snake_case   	=      [4, 1_0_7, 3_8, 2_0_1, 1_7_4, 3_8, 2_0_1, 1_7_5, 3_8, 2_0_1, 1_7_6, 3_8, 2_0_1, 1_7_7, 5]
     self.assertEqual(encoded['''input_ids''']							,       __lowerCamelCase				)
     # decoding
     _snake_case   	=      tokenizer.decode(__lowerCamelCase				)
     self.assertEqual(__lowerCamelCase							,       '''[CLS]e è é ê ë[SEP]'''				)

     # encode/decode, but with `encode` instead of `__call__`
     self.assertEqual(tokenizer.decode(tokenizer.encode('''e è é ê ë'''				)				)							,       '''[CLS]e è é ê ë[SEP]'''				)



   def    __UpperCAmelCase    (       self    :	Optional[int]				):



     """simple docstring"""
     _snake_case   	=      self.perceiver_tokenizer
     _snake_case   	=      ['''A long paragraph for summarization.''', '''Another paragraph for summarization.''']
     # fmt: off
     _snake_case   	=      [4, 7_1, 3_8, 1_1_4, 1_1_7, 1_1_6, 1_0_9, 3_8, 1_1_8, 1_0_3, 1_2_0, 1_0_3, 1_0_9, 1_2_0, 1_0_3, 1_1_8, 1_1_0, 3_8, 1_0_8, 1_1_7, 1_2_0, 3_8, 1_2_1, 1_2_3, 1_1_5, 1_1_5, 1_0_3, 1_2_0, 1_1_1, 1_2_8, 1_0_3, 1_2_2, 1_1_1, 1_1_7, 1_1_6, 5_2, 5, 0]
     # fmt: on
     _snake_case   	=      tokenizer(__lowerCamelCase							,       padding=__lowerCamelCase							,       return_tensors=__lowerCamelCase				)
     self.assertIsInstance(__lowerCamelCase							,       __lowerCamelCase				)

     if FRAMEWORK != "jax":
       _snake_case   	=      list(batch.input_ids.numpy()[0]				)
     else:
       _snake_case   	=      list(batch.input_ids.tolist()[0]				)

     self.assertListEqual(__lowerCamelCase							,       __lowerCamelCase				)

     self.assertEqual((2, 3_8)							,       batch.input_ids.shape				)
     self.assertEqual((2, 3_8)							,       batch.attention_mask.shape				)



   def    __UpperCAmelCase    (       self    :	Any				):



     """simple docstring"""
     _snake_case   	=      self.perceiver_tokenizer
     _snake_case   	=      ['''A long paragraph for summarization.''', '''Another paragraph for summarization.''']
     _snake_case   	=      tokenizer(__lowerCamelCase							,       padding=__lowerCamelCase							,       return_tensors=__lowerCamelCase				)
     # check if input_ids are returned and no decoder_input_ids
     self.assertIn('''input_ids'''							,       __lowerCamelCase				)
     self.assertIn('''attention_mask'''							,       __lowerCamelCase				)
     self.assertNotIn('''decoder_input_ids'''							,       __lowerCamelCase				)
     self.assertNotIn('''decoder_attention_mask'''							,       __lowerCamelCase				)



   def    __UpperCAmelCase    (       self    :	int				):



     """simple docstring"""
     _snake_case   	=      self.perceiver_tokenizer
     _snake_case   	=      [
         '''Summary of the text.''',
         '''Another summary.''',
     ]
     _snake_case   	=      tokenizer(
         text_target=__lowerCamelCase							,       max_length=3_2							,       padding='''max_length'''							,       truncation=__lowerCamelCase							,       return_tensors=__lowerCamelCase				)
     self.assertEqual(3_2							,       targets['''input_ids'''].shape[1]				)



   def    __UpperCAmelCase    (       self    :	List[Any]				):



     """simple docstring"""
     # safety check on max_len default value so we are sure the test works
     _snake_case   	=      self.get_tokenizers()
     for tokenizer in tokenizers:
       with self.subTest(f"""{tokenizer.__class__.__name__}"""				):
         self.assertNotEqual(tokenizer.model_max_length							,       4_2				)

        # Now let's start the test
     _snake_case   	=      self.get_tokenizers()
     for tokenizer in tokenizers:
       with self.subTest(f"""{tokenizer.__class__.__name__}"""				):
         # Isolate this from the other tests because we save additional tokens/etc
         _snake_case   	=      tempfile.mkdtemp()

         _snake_case   	=      ''' He is very happy, UNwant\u00E9d,running'''
         _snake_case   	=      tokenizer.encode(__lowerCamelCase							,       add_special_tokens=__lowerCamelCase				)
         tokenizer.save_pretrained(__lowerCamelCase				)

         _snake_case   	=      tokenizer.__class__.from_pretrained(__lowerCamelCase				)
         _snake_case   	=      after_tokenizer.encode(__lowerCamelCase							,       add_special_tokens=__lowerCamelCase				)
         self.assertListEqual(__lowerCamelCase							,       __lowerCamelCase				)

         shutil.rmtree(__lowerCamelCase				)

     _snake_case   	=      self.get_tokenizers(model_max_length=4_2				)
     for tokenizer in tokenizers:
       with self.subTest(f"""{tokenizer.__class__.__name__}"""				):
         # Isolate this from the other tests because we save additional tokens/etc
         _snake_case   	=      tempfile.mkdtemp()

         _snake_case   	=      ''' He is very happy, UNwant\u00E9d,running'''
         tokenizer.add_tokens(['''bim''', '''bambam''']				)
         _snake_case   	=      tokenizer.additional_special_tokens
         additional_special_tokens.append('''new_additional_special_token'''				)
         tokenizer.add_special_tokens({'''additional_special_tokens''': additional_special_tokens}				)
         _snake_case   	=      tokenizer.encode(__lowerCamelCase							,       add_special_tokens=__lowerCamelCase				)
         tokenizer.save_pretrained(__lowerCamelCase				)

         _snake_case   	=      tokenizer.__class__.from_pretrained(__lowerCamelCase				)
         _snake_case   	=      after_tokenizer.encode(__lowerCamelCase							,       add_special_tokens=__lowerCamelCase				)
         self.assertListEqual(__lowerCamelCase							,       __lowerCamelCase				)
         self.assertIn('''new_additional_special_token'''							,       after_tokenizer.additional_special_tokens				)
         self.assertEqual(after_tokenizer.model_max_length							,       4_2				)

         _snake_case   	=      tokenizer.__class__.from_pretrained(__lowerCamelCase							,       model_max_length=4_3				)
         self.assertEqual(tokenizer.model_max_length							,       4_3				)

         shutil.rmtree(__lowerCamelCase				)



   def    __UpperCAmelCase    (       self    :	Tuple				):



     """simple docstring"""
     _snake_case   	=      []
     if self.test_slow_tokenizer:
       tokenizer_list.append((self.tokenizer_class, self.get_tokenizer())				)

     if self.test_rust_tokenizer:
       tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer())				)

     for tokenizer_class, tokenizer_utils in tokenizer_list:
       with tempfile.TemporaryDirectory() as tmp_dir:
         tokenizer_utils.save_pretrained(__lowerCamelCase				)

         with open(os.path.join(__lowerCamelCase							,       '''special_tokens_map.json'''				)							,       encoding='''utf-8'''				) as json_file:
           _snake_case   	=      json.load(__lowerCamelCase				)

         with open(os.path.join(__lowerCamelCase							,       '''tokenizer_config.json'''				)							,       encoding='''utf-8'''				) as json_file:
           _snake_case   	=      json.load(__lowerCamelCase				)

         _snake_case   	=      [f"""<extra_id_{i}>""" for i in range(1_2_5				)]

         _snake_case   	=      added_tokens_extra_ids + [
             '''an_additional_special_token'''
         ]
         _snake_case   	=      added_tokens_extra_ids + [
             '''an_additional_special_token'''
         ]

         with open(os.path.join(__lowerCamelCase							,       '''special_tokens_map.json'''				)							,       '''w'''							,       encoding='''utf-8'''				) as outfile:
           json.dump(__lowerCamelCase							,       __lowerCamelCase				)
         with open(os.path.join(__lowerCamelCase							,       '''tokenizer_config.json'''				)							,       '''w'''							,       encoding='''utf-8'''				) as outfile:
           json.dump(__lowerCamelCase							,       __lowerCamelCase				)

         # the following checks allow us to verify that our test works as expected, i.e. that the tokenizer takes
         # into account the new value of additional_special_tokens given in the "tokenizer_config.json" and
         # "special_tokens_map.json" files
         _snake_case   	=      tokenizer_class.from_pretrained(
             __lowerCamelCase							,       )
         self.assertIn(
             '''an_additional_special_token'''							,       tokenizer_without_change_in_init.additional_special_tokens				)
         self.assertEqual(
             ['''an_additional_special_token''']							,       tokenizer_without_change_in_init.convert_ids_to_tokens(
                 tokenizer_without_change_in_init.convert_tokens_to_ids(['''an_additional_special_token''']				)				)							,       )

         # Now we test that we can change the value of additional_special_tokens in the from_pretrained
         _snake_case   	=      added_tokens_extra_ids + [AddedToken('''a_new_additional_special_token'''							,       lstrip=__lowerCamelCase				)]
         _snake_case   	=      tokenizer_class.from_pretrained(
             __lowerCamelCase							,       additional_special_tokens=__lowerCamelCase							,       )

         self.assertIn('''a_new_additional_special_token'''							,       tokenizer.additional_special_tokens				)
         self.assertEqual(
             ['''a_new_additional_special_token''']							,       tokenizer.convert_ids_to_tokens(
                 tokenizer.convert_tokens_to_ids(['''a_new_additional_special_token''']				)				)							,       )



   def    __UpperCAmelCase    (       self    :	Optional[int]				):



     """simple docstring"""
     _snake_case   	=      self.perceiver_tokenizer
     self.assertEqual(tokenizer.decode([1_7_8]				)							,       '''�'''				)



   def    __UpperCAmelCase    (       self    :	Any				):



     """simple docstring"""
     pass



   def    __UpperCAmelCase    (       self    :	str				):



     """simple docstring"""
     pass



   def    __UpperCAmelCase    (       self    :	str				):



     """simple docstring"""
     pass



   def    __UpperCAmelCase    (       self    :	int				):



     """simple docstring"""
     pass





   def    __UpperCAmelCase    (       self    :	List[Any]				):



     """simple docstring"""
     # The default common tokenizer tests uses invalid tokens for Perceiver that can only accept one-character
     # strings and special added tokens as tokens
     _snake_case   	=      self.get_tokenizers(fast=__lowerCamelCase							,       do_lower_case=__lowerCamelCase				)
     for tokenizer in tokenizers:
       with self.subTest(f"""{tokenizer.__class__.__name__}"""				):
         _snake_case   	=      ['''[CLS]''', '''t''', '''h''', '''i''', '''s''', ''' ''', '''i''', '''s''', ''' ''', '''a''', ''' ''', '''t''', '''e''', '''s''', '''t''', '''[SEP]''']
         _snake_case   	=      tokenizer.convert_tokens_to_string(__lowerCamelCase				)

         self.assertIsInstance(__lowerCamelCase							,       __lowerCamelCase				)





