import json
from typing import List, Optional, Tuple

from tokenizers import pre_tokenizers, processors

from ...tokenization_utils_base import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_roberta import RobertaTokenizer


lowerCamelCase__  =					logging.get_logger(__name__)

lowerCamelCase__  =					{'''vocab_file''': '''vocab.json''', '''merges_file''': '''merges.txt''', '''tokenizer_file''': '''tokenizer.json'''}

lowerCamelCase__  =					{
    '''vocab_file''': {
        '''roberta-base''': '''https://huggingface.co/roberta-base/resolve/main/vocab.json''',
        '''roberta-large''': '''https://huggingface.co/roberta-large/resolve/main/vocab.json''',
        '''roberta-large-mnli''': '''https://huggingface.co/roberta-large-mnli/resolve/main/vocab.json''',
        '''distilroberta-base''': '''https://huggingface.co/distilroberta-base/resolve/main/vocab.json''',
        '''roberta-base-openai-detector''': '''https://huggingface.co/roberta-base-openai-detector/resolve/main/vocab.json''',
        '''roberta-large-openai-detector''': (
            '''https://huggingface.co/roberta-large-openai-detector/resolve/main/vocab.json'''
        ),
    },
    '''merges_file''': {
        '''roberta-base''': '''https://huggingface.co/roberta-base/resolve/main/merges.txt''',
        '''roberta-large''': '''https://huggingface.co/roberta-large/resolve/main/merges.txt''',
        '''roberta-large-mnli''': '''https://huggingface.co/roberta-large-mnli/resolve/main/merges.txt''',
        '''distilroberta-base''': '''https://huggingface.co/distilroberta-base/resolve/main/merges.txt''',
        '''roberta-base-openai-detector''': '''https://huggingface.co/roberta-base-openai-detector/resolve/main/merges.txt''',
        '''roberta-large-openai-detector''': (
            '''https://huggingface.co/roberta-large-openai-detector/resolve/main/merges.txt'''
        ),
    },
    '''tokenizer_file''': {
        '''roberta-base''': '''https://huggingface.co/roberta-base/resolve/main/tokenizer.json''',
        '''roberta-large''': '''https://huggingface.co/roberta-large/resolve/main/tokenizer.json''',
        '''roberta-large-mnli''': '''https://huggingface.co/roberta-large-mnli/resolve/main/tokenizer.json''',
        '''distilroberta-base''': '''https://huggingface.co/distilroberta-base/resolve/main/tokenizer.json''',
        '''roberta-base-openai-detector''': (
            '''https://huggingface.co/roberta-base-openai-detector/resolve/main/tokenizer.json'''
        ),
        '''roberta-large-openai-detector''': (
            '''https://huggingface.co/roberta-large-openai-detector/resolve/main/tokenizer.json'''
        ),
    },
}

lowerCamelCase__  =					{
    '''roberta-base''': 512,
    '''roberta-large''': 512,
    '''roberta-large-mnli''': 512,
    '''distilroberta-base''': 512,
    '''roberta-base-openai-detector''': 512,
    '''roberta-large-openai-detector''': 512,
}




class 					_UpperCAmelCase			(     lowerCAmelCase  ):



 '''simple docstring'''




 __A			       =					VOCAB_FILES_NAMES
 __A			       =					PRETRAINED_VOCAB_FILES_MAP
 __A			       =					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 __A			       =					['''input_ids''', '''attention_mask''']
 __A			       =					RobertaTokenizer







 def __init__(		self				:   List[Any]							, lowercase_				:   Optional[Any]=None							, lowercase_				:   Optional[Any]=None							, lowercase_				:   Optional[Any]=None							, lowercase_				:   List[str]="replace"							, lowercase_				:   Optional[Any]="<s>"							, lowercase_				:   int="</s>"							, lowercase_				:   Dict="</s>"							, lowercase_				:   Optional[Any]="<s>"							, lowercase_				:   Any="<unk>"							, lowercase_				:   Tuple="<pad>"							, lowercase_				:   Dict="<mask>"							, lowercase_				:   int=False							, lowercase_				:   List[Any]=True							, **lowercase_				:   Optional[int]							, )						->	Optional[Any]:


   """simple docstring"""







   super().__init__(
       lowercase_							, lowercase_							, tokenizer_file=lowercase_							, errors=lowercase_							, bos_token=lowercase_							, eos_token=lowercase_							, sep_token=lowercase_							, cls_token=lowercase_							, unk_token=lowercase_							, pad_token=lowercase_							, mask_token=lowercase_							, add_prefix_space=lowercase_							, trim_offsets=lowercase_							, **lowercase_							, )

   _UpperCamelCase           =    json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__())
   if pre_tok_state.get("add_prefix_space"							, lowercase_) != add_prefix_space:
     _UpperCamelCase           =    getattr(lowercase_							, pre_tok_state.pop("type"))
     _UpperCamelCase           =    add_prefix_space
     _UpperCamelCase           =    pre_tok_class(**lowercase_)

   _UpperCamelCase           =    add_prefix_space

   _UpperCamelCase           =    "post_processor"
   _UpperCamelCase           =    getattr(self.backend_tokenizer							, lowercase_							, lowercase_)
   if tokenizer_component_instance:
     _UpperCamelCase           =    json.loads(tokenizer_component_instance.__getstate__())

     # The lists 'sep' and 'cls' must be cased in tuples for the object `post_processor_class`
     if "sep" in state:
       _UpperCamelCase           =    tuple(state["sep"])
     if "cls" in state:
       _UpperCamelCase           =    tuple(state["cls"])

     _UpperCamelCase           =    False

     if state.get("add_prefix_space"							, lowercase_) != add_prefix_space:
       _UpperCamelCase           =    add_prefix_space
       _UpperCamelCase           =    True

     if state.get("trim_offsets"							, lowercase_) != trim_offsets:
       _UpperCamelCase           =    trim_offsets
       _UpperCamelCase           =    True

     if changes_to_apply:
       _UpperCamelCase           =    getattr(lowercase_							, state.pop("type"))
       _UpperCamelCase           =    component_class(**lowercase_)
       setattr(self.backend_tokenizer							, lowercase_							, lowercase_)







 @property
 def 		__UpperCAmelCase				(		self				:   int)						->	str:


   """simple docstring"""







   if self._mask_token is None:
     if self.verbose:
       logger.error("Using mask_token, but it is not set yet.")
     return None
   return str(self._mask_token)







 @mask_token.setter
 def 		__UpperCAmelCase				(		self				:   List[Any]							, lowercase_				:   Optional[Any])						->	Dict:


   """simple docstring"""







   _UpperCamelCase           =    AddedToken(lowercase_							, lstrip=lowercase_							, rstrip=lowercase_) if isinstance(lowercase_							, lowercase_) else value
   _UpperCamelCase           =    value







 def 		__UpperCAmelCase				(		self				:   Optional[int]							, *lowercase_				:   List[str]							, **lowercase_				:   int)						->	BatchEncoding:


   """simple docstring"""







   _UpperCamelCase           =    kwargs.get("is_split_into_words"							, lowercase_)
   assert self.add_prefix_space or not is_split_into_words, (
       f'You need to instantiate {self.__class__.__name__} with add_prefix_space=True '
       "to use it with pretokenized inputs."
   )

   return super()._batch_encode_plus(*lowercase_							, **lowercase_)







 def 		__UpperCAmelCase				(		self				:   str							, *lowercase_				:   Optional[int]							, **lowercase_				:   Dict)						->	BatchEncoding:


   """simple docstring"""







   _UpperCamelCase           =    kwargs.get("is_split_into_words"							, lowercase_)

   assert self.add_prefix_space or not is_split_into_words, (
       f'You need to instantiate {self.__class__.__name__} with add_prefix_space=True '
       "to use it with pretokenized inputs."
   )

   return super()._encode_plus(*lowercase_							, **lowercase_)







 def 		__UpperCAmelCase				(		self				:   str							, lowercase_				:   str							, lowercase_				:   Optional[str] = None)						->	Tuple[str]:


   """simple docstring"""







   _UpperCamelCase           =    self._tokenizer.model.save(lowercase_							, name=lowercase_)
   return tuple(lowercase_)







 def 		__UpperCAmelCase				(		self				:   Dict							, lowercase_				:   Dict							, lowercase_				:   str=None)						->	Union[str, Any]:


   """simple docstring"""







   _UpperCamelCase           =    [self.bos_token_id] + token_ids_a + [self.eos_token_id]
   if token_ids_a is None:
     return output

   return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]







 def 		__UpperCAmelCase				(		self				:   List[str]							, lowercase_				:   List[int]							, lowercase_				:   Optional[List[int]] = None)						->	List[int]:


   """simple docstring"""







   _UpperCamelCase           =    [self.sep_token_id]
   _UpperCamelCase           =    [self.cls_token_id]

   if token_ids_a is None:
     return len(cls + token_ids_a + sep) * [0]
   return len(cls + token_ids_a + sep + sep + token_ids_a + sep) * [0]





import unittest

from transformers import BigBirdConfig, is_flax_available
from transformers.testing_utils import require_flax, slow

from ...test_modeling_flax_common import FlaxModelTesterMixin, ids_tensor, random_attention_mask


if is_flax_available():
   import jax

   from transformers.models.big_bird.modeling_flax_big_bird import (
       FlaxBigBirdForCausalLM,
       FlaxBigBirdForMaskedLM,
       FlaxBigBirdForMultipleChoice,
       FlaxBigBirdForPreTraining,
       FlaxBigBirdForQuestionAnswering,
       FlaxBigBirdForSequenceClassification,
       FlaxBigBirdForTokenClassification,
       FlaxBigBirdModel,
   )




class 					_UpperCAmelCase			(     unittest.TestCase  ):



 '''simple docstring'''




 def __init__(		self				:   List[Any]							, lowercase_				:   Union[str, Any]							, lowercase_				:   Union[str, Any]=2							, lowercase_				:   Dict=56							, lowercase_				:   str=True							, lowercase_				:   Optional[Any]=True							, lowercase_				:   List[Any]=True							, lowercase_				:   Union[str, Any]=True							, lowercase_				:   Any=99							, lowercase_				:   Optional[int]=32							, lowercase_				:   Tuple=2							, lowercase_				:   int=2							, lowercase_				:   List[str]=7							, lowercase_				:   Any="gelu_new"							, lowercase_				:   List[str]=0.1							, lowercase_				:   str=0.1							, lowercase_				:   List[Any]=512							, lowercase_				:   List[str]=16							, lowercase_				:   Optional[int]=2							, lowercase_				:   Union[str, Any]=0.02							, lowercase_				:   Union[str, Any]=4							, lowercase_				:   Union[str, Any]="block_sparse"							, lowercase_				:   Tuple=True							, lowercase_				:   Dict=False							, lowercase_				:   Dict=2							, lowercase_				:   Dict=3							, )						->	List[str]:


   """simple docstring"""







   _UpperCamelCase           =    parent
   _UpperCamelCase           =    batch_size
   _UpperCamelCase           =    seq_length
   _UpperCamelCase           =    is_training
   _UpperCamelCase           =    use_attention_mask
   _UpperCamelCase           =    use_token_type_ids
   _UpperCamelCase           =    use_labels
   _UpperCamelCase           =    vocab_size
   _UpperCamelCase           =    hidden_size
   _UpperCamelCase           =    num_hidden_layers
   _UpperCamelCase           =    num_attention_heads
   _UpperCamelCase           =    intermediate_size
   _UpperCamelCase           =    hidden_act
   _UpperCamelCase           =    hidden_dropout_prob
   _UpperCamelCase           =    attention_probs_dropout_prob
   _UpperCamelCase           =    max_position_embeddings
   _UpperCamelCase           =    type_vocab_size
   _UpperCamelCase           =    type_sequence_label_size
   _UpperCamelCase           =    initializer_range
   _UpperCamelCase           =    num_choices

   _UpperCamelCase           =    rescale_embeddings
   _UpperCamelCase           =    attention_type
   _UpperCamelCase           =    use_bias
   _UpperCamelCase           =    block_size
   _UpperCamelCase           =    num_random_blocks







 def 		__UpperCAmelCase				(		self				:   List[Any])						->	Dict:


   """simple docstring"""







   _UpperCamelCase           =    ids_tensor([self.batch_size, self.seq_length]							, self.vocab_size)

   _UpperCamelCase           =    None
   if self.use_attention_mask:
     _UpperCamelCase           =    random_attention_mask([self.batch_size, self.seq_length])

   _UpperCamelCase           =    None
   if self.use_token_type_ids:
     _UpperCamelCase           =    ids_tensor([self.batch_size, self.seq_length]							, self.type_vocab_size)

   _UpperCamelCase           =    BigBirdConfig(
       vocab_size=self.vocab_size							, hidden_size=self.hidden_size							, num_hidden_layers=self.num_hidden_layers							, num_attention_heads=self.num_attention_heads							, intermediate_size=self.intermediate_size							, hidden_act=self.hidden_act							, hidden_dropout_prob=self.hidden_dropout_prob							, attention_probs_dropout_prob=self.attention_probs_dropout_prob							, max_position_embeddings=self.max_position_embeddings							, type_vocab_size=self.type_vocab_size							, is_decoder=lowercase_							, initializer_range=self.initializer_range							, attention_type=self.attention_type							, block_size=self.block_size							, num_random_blocks=self.num_random_blocks							, use_bias=self.use_bias							, rescale_embeddings=self.rescale_embeddings							, )

   return config, input_ids, token_type_ids, attention_mask







 def 		__UpperCAmelCase				(		self				:   Optional[Any])						->	Optional[Any]:


   """simple docstring"""







   _UpperCamelCase           =    self.prepare_config_and_inputs()
   _UpperCamelCase      ,						_UpperCamelCase      ,						_UpperCamelCase      ,						_UpperCamelCase           =    config_and_inputs
   _UpperCamelCase           =    {
       "input_ids": input_ids,
       "token_type_ids": token_type_ids,
       "attention_mask": attention_mask,
   }
   return config, inputs_dict




@require_flax
class 					_UpperCAmelCase			(     lowerCAmelCase,			unittest.TestCase  ):



 '''simple docstring'''




 __A			       =					(
     (
         FlaxBigBirdForCausalLM,
         FlaxBigBirdModel,
         FlaxBigBirdForPreTraining,
         FlaxBigBirdForMaskedLM,
         FlaxBigBirdForMultipleChoice,
         FlaxBigBirdForQuestionAnswering,
         FlaxBigBirdForSequenceClassification,
         FlaxBigBirdForTokenClassification,
     )
     if is_flax_available()
     else ()
 )

 __A			       =					False
 __A			       =					False







 def 		__UpperCAmelCase				(		self				:   Optional[int])						->	Optional[Any]:


   """simple docstring"""







   _UpperCamelCase           =    FlaxBigBirdModelTester(self)







 @slow
 # copied from `test_modeling_flax_common` because it takes much longer than other models
 def 		__UpperCAmelCase				(		self				:   Union[str, Any])						->	List[str]:


   """simple docstring"""







   super().test_from_pretrained_save_pretrained()







 @slow
 # copied from `test_modeling_flax_common` because it takes much longer than other models
 def 		__UpperCAmelCase				(		self				:   Any)						->	int:


   """simple docstring"""







   super().test_from_pretrained_with_no_automatic_init()







 @slow
 # copied from `test_modeling_flax_common` because it takes much longer than other models
 def 		__UpperCAmelCase				(		self				:   List[Any])						->	Union[str, Any]:


   """simple docstring"""







   super().test_no_automatic_init()







 @slow
 # copied from `test_modeling_flax_common` because it takes much longer than other models
 def 		__UpperCAmelCase				(		self				:   str)						->	Union[str, Any]:


   """simple docstring"""







   super().test_hidden_states_output()







 @slow
 def 		__UpperCAmelCase				(		self				:   Tuple)						->	Optional[int]:


   """simple docstring"""







   for model_class_name in self.all_model_classes:
     _UpperCamelCase           =    model_class_name.from_pretrained("google/bigbird-roberta-base")
     self.assertIsNotNone(lowercase_)







 def 		__UpperCAmelCase				(		self				:   Tuple)						->	str:


   """simple docstring"""







   if self.test_attn_probs:
     super().test_attention_outputs()







 @slow
 # copied from `test_modeling_flax_common` because it takes much longer than other models
 def 		__UpperCAmelCase				(		self				:   Tuple)						->	Dict:


   """simple docstring"""







   _UpperCamelCase      ,						_UpperCamelCase           =    self.model_tester.prepare_config_and_inputs_for_common()

   for model_class in self.all_model_classes:
     with self.subTest(model_class.__name__):
       _UpperCamelCase           =    self._prepare_for_class(lowercase_							, lowercase_)
       _UpperCamelCase           =    model_class(lowercase_)

       @jax.jit
       def model_jitted(lowercase_				:   Dict							, lowercase_				:   List[Any]=None							, **lowercase_				:   Tuple):
         return model(input_ids=lowercase_							, attention_mask=lowercase_							, **lowercase_)

       with self.subTest("JIT Enabled"):
         _UpperCamelCase           =    model_jitted(**lowercase_).to_tuple()

       with self.subTest("JIT Disabled"):
         with jax.disable_jit():
           _UpperCamelCase           =    model_jitted(**lowercase_).to_tuple()

       self.assertEqual(len(lowercase_)							, len(lowercase_))
       for jitted_output, output in zip(lowercase_							, lowercase_):
         self.assertEqual(jitted_output.shape							, output.shape)







 def 		__UpperCAmelCase				(		self				:   Any							, lowercase_				:   List[str]							, lowercase_				:   List[str]							, lowercase_				:   List[str]							, lowercase_				:   str=1e-5							, lowercase_				:   int="outputs"							, lowercase_				:   List[str]=None)						->	Tuple:


   """simple docstring"""







   if name.startswith("outputs.attentions"):
     return
   else:
     super().check_pt_flax_outputs(lowercase_							, lowercase_							, lowercase_							, lowercase_							, lowercase_							, lowercase_)




