
"""simple docstring"""







import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_bert import BertTokenizer


lowerCamelCase__           =	logging.get_logger(__name__)

lowerCamelCase__           =	{"""vocab_file""": """vocab.txt""", """tokenizer_file""": """tokenizer.json"""}

lowerCamelCase__           =	{
    """vocab_file""": {
        """bert-base-uncased""": """https://huggingface.co/bert-base-uncased/resolve/main/vocab.txt""",
        """bert-large-uncased""": """https://huggingface.co/bert-large-uncased/resolve/main/vocab.txt""",
        """bert-base-cased""": """https://huggingface.co/bert-base-cased/resolve/main/vocab.txt""",
        """bert-large-cased""": """https://huggingface.co/bert-large-cased/resolve/main/vocab.txt""",
        """bert-base-multilingual-uncased""": (
            """https://huggingface.co/bert-base-multilingual-uncased/resolve/main/vocab.txt"""
        ),
        """bert-base-multilingual-cased""": """https://huggingface.co/bert-base-multilingual-cased/resolve/main/vocab.txt""",
        """bert-base-chinese""": """https://huggingface.co/bert-base-chinese/resolve/main/vocab.txt""",
        """bert-base-german-cased""": """https://huggingface.co/bert-base-german-cased/resolve/main/vocab.txt""",
        """bert-large-uncased-whole-word-masking""": (
            """https://huggingface.co/bert-large-uncased-whole-word-masking/resolve/main/vocab.txt"""
        ),
        """bert-large-cased-whole-word-masking""": (
            """https://huggingface.co/bert-large-cased-whole-word-masking/resolve/main/vocab.txt"""
        ),
        """bert-large-uncased-whole-word-masking-finetuned-squad""": (
            """https://huggingface.co/bert-large-uncased-whole-word-masking-finetuned-squad/resolve/main/vocab.txt"""
        ),
        """bert-large-cased-whole-word-masking-finetuned-squad""": (
            """https://huggingface.co/bert-large-cased-whole-word-masking-finetuned-squad/resolve/main/vocab.txt"""
        ),
        """bert-base-cased-finetuned-mrpc""": (
            """https://huggingface.co/bert-base-cased-finetuned-mrpc/resolve/main/vocab.txt"""
        ),
        """bert-base-german-dbmdz-cased""": """https://huggingface.co/bert-base-german-dbmdz-cased/resolve/main/vocab.txt""",
        """bert-base-german-dbmdz-uncased""": (
            """https://huggingface.co/bert-base-german-dbmdz-uncased/resolve/main/vocab.txt"""
        ),
        """TurkuNLP/bert-base-finnish-cased-v1""": (
            """https://huggingface.co/TurkuNLP/bert-base-finnish-cased-v1/resolve/main/vocab.txt"""
        ),
        """TurkuNLP/bert-base-finnish-uncased-v1""": (
            """https://huggingface.co/TurkuNLP/bert-base-finnish-uncased-v1/resolve/main/vocab.txt"""
        ),
        """wietsedv/bert-base-dutch-cased""": (
            """https://huggingface.co/wietsedv/bert-base-dutch-cased/resolve/main/vocab.txt"""
        ),
    },
    """tokenizer_file""": {
        """bert-base-uncased""": """https://huggingface.co/bert-base-uncased/resolve/main/tokenizer.json""",
        """bert-large-uncased""": """https://huggingface.co/bert-large-uncased/resolve/main/tokenizer.json""",
        """bert-base-cased""": """https://huggingface.co/bert-base-cased/resolve/main/tokenizer.json""",
        """bert-large-cased""": """https://huggingface.co/bert-large-cased/resolve/main/tokenizer.json""",
        """bert-base-multilingual-uncased""": (
            """https://huggingface.co/bert-base-multilingual-uncased/resolve/main/tokenizer.json"""
        ),
        """bert-base-multilingual-cased""": (
            """https://huggingface.co/bert-base-multilingual-cased/resolve/main/tokenizer.json"""
        ),
        """bert-base-chinese""": """https://huggingface.co/bert-base-chinese/resolve/main/tokenizer.json""",
        """bert-base-german-cased""": """https://huggingface.co/bert-base-german-cased/resolve/main/tokenizer.json""",
        """bert-large-uncased-whole-word-masking""": (
            """https://huggingface.co/bert-large-uncased-whole-word-masking/resolve/main/tokenizer.json"""
        ),
        """bert-large-cased-whole-word-masking""": (
            """https://huggingface.co/bert-large-cased-whole-word-masking/resolve/main/tokenizer.json"""
        ),
        """bert-large-uncased-whole-word-masking-finetuned-squad""": (
            """https://huggingface.co/bert-large-uncased-whole-word-masking-finetuned-squad/resolve/main/tokenizer.json"""
        ),
        """bert-large-cased-whole-word-masking-finetuned-squad""": (
            """https://huggingface.co/bert-large-cased-whole-word-masking-finetuned-squad/resolve/main/tokenizer.json"""
        ),
        """bert-base-cased-finetuned-mrpc""": (
            """https://huggingface.co/bert-base-cased-finetuned-mrpc/resolve/main/tokenizer.json"""
        ),
        """bert-base-german-dbmdz-cased""": (
            """https://huggingface.co/bert-base-german-dbmdz-cased/resolve/main/tokenizer.json"""
        ),
        """bert-base-german-dbmdz-uncased""": (
            """https://huggingface.co/bert-base-german-dbmdz-uncased/resolve/main/tokenizer.json"""
        ),
        """TurkuNLP/bert-base-finnish-cased-v1""": (
            """https://huggingface.co/TurkuNLP/bert-base-finnish-cased-v1/resolve/main/tokenizer.json"""
        ),
        """TurkuNLP/bert-base-finnish-uncased-v1""": (
            """https://huggingface.co/TurkuNLP/bert-base-finnish-uncased-v1/resolve/main/tokenizer.json"""
        ),
        """wietsedv/bert-base-dutch-cased""": (
            """https://huggingface.co/wietsedv/bert-base-dutch-cased/resolve/main/tokenizer.json"""
        ),
    },
}

lowerCamelCase__           =	{
    """bert-base-uncased""": 512,
    """bert-large-uncased""": 512,
    """bert-base-cased""": 512,
    """bert-large-cased""": 512,
    """bert-base-multilingual-uncased""": 512,
    """bert-base-multilingual-cased""": 512,
    """bert-base-chinese""": 512,
    """bert-base-german-cased""": 512,
    """bert-large-uncased-whole-word-masking""": 512,
    """bert-large-cased-whole-word-masking""": 512,
    """bert-large-uncased-whole-word-masking-finetuned-squad""": 512,
    """bert-large-cased-whole-word-masking-finetuned-squad""": 512,
    """bert-base-cased-finetuned-mrpc""": 512,
    """bert-base-german-dbmdz-cased""": 512,
    """bert-base-german-dbmdz-uncased""": 512,
    """TurkuNLP/bert-base-finnish-cased-v1""": 512,
    """TurkuNLP/bert-base-finnish-uncased-v1""": 512,
    """wietsedv/bert-base-dutch-cased""": 512,
}

lowerCamelCase__           =	{
    """bert-base-uncased""": {"""do_lower_case""": True},
    """bert-large-uncased""": {"""do_lower_case""": True},
    """bert-base-cased""": {"""do_lower_case""": False},
    """bert-large-cased""": {"""do_lower_case""": False},
    """bert-base-multilingual-uncased""": {"""do_lower_case""": True},
    """bert-base-multilingual-cased""": {"""do_lower_case""": False},
    """bert-base-chinese""": {"""do_lower_case""": False},
    """bert-base-german-cased""": {"""do_lower_case""": False},
    """bert-large-uncased-whole-word-masking""": {"""do_lower_case""": True},
    """bert-large-cased-whole-word-masking""": {"""do_lower_case""": False},
    """bert-large-uncased-whole-word-masking-finetuned-squad""": {"""do_lower_case""": True},
    """bert-large-cased-whole-word-masking-finetuned-squad""": {"""do_lower_case""": False},
    """bert-base-cased-finetuned-mrpc""": {"""do_lower_case""": False},
    """bert-base-german-dbmdz-cased""": {"""do_lower_case""": False},
    """bert-base-german-dbmdz-uncased""": {"""do_lower_case""": True},
    """TurkuNLP/bert-base-finnish-cased-v1""": {"""do_lower_case""": False},
    """TurkuNLP/bert-base-finnish-uncased-v1""": {"""do_lower_case""": True},
    """wietsedv/bert-base-dutch-cased""": {"""do_lower_case""": False},
}




class 				A__ (			_lowerCamelCase):
		A_		: Tuple            =			VOCAB_FILES_NAMES
		A_		: List[str]            =			PRETRAINED_VOCAB_FILES_MAP
		A_		: List[Any]            =			PRETRAINED_INIT_CONFIGURATION
		A_		: Optional[Any]            =			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
		A_		: Optional[Any]            =			BertTokenizer

		def __init__(   self ,	_SCREAMING_SNAKE_CASE=None ,	_SCREAMING_SNAKE_CASE=None ,	_SCREAMING_SNAKE_CASE=True ,	_SCREAMING_SNAKE_CASE="[UNK]" ,	_SCREAMING_SNAKE_CASE="[SEP]" ,	_SCREAMING_SNAKE_CASE="[PAD]" ,	_SCREAMING_SNAKE_CASE="[CLS]" ,	_SCREAMING_SNAKE_CASE="[MASK]" ,	_SCREAMING_SNAKE_CASE=True ,	_SCREAMING_SNAKE_CASE=None ,	**_SCREAMING_SNAKE_CASE ,	):
			super().__init__(
			    _SCREAMING_SNAKE_CASE ,	tokenizer_file=_SCREAMING_SNAKE_CASE ,	do_lower_case=_SCREAMING_SNAKE_CASE ,	unk_token=_SCREAMING_SNAKE_CASE ,	sep_token=_SCREAMING_SNAKE_CASE ,	pad_token=_SCREAMING_SNAKE_CASE ,	cls_token=_SCREAMING_SNAKE_CASE ,	mask_token=_SCREAMING_SNAKE_CASE ,	tokenize_chinese_chars=_SCREAMING_SNAKE_CASE ,	strip_accents=_SCREAMING_SNAKE_CASE ,	**_SCREAMING_SNAKE_CASE ,	)

			__lowerCAmelCase    :		Optional[Any]		    =   json.loads(self.backend_tokenizer.normalizer.__getstate__()				)
			if (
			    normalizer_state.get('lowercase' ,	_SCREAMING_SNAKE_CASE				) != do_lower_case
			    or normalizer_state.get('strip_accents' ,	_SCREAMING_SNAKE_CASE				) != strip_accents
			    or normalizer_state.get('handle_chinese_chars' ,	_SCREAMING_SNAKE_CASE				) != tokenize_chinese_chars
			):
				__lowerCAmelCase    :		List[Any]		    =   getattr(_SCREAMING_SNAKE_CASE ,	normalizer_state.pop('type'				)				)
				__lowerCAmelCase    :		List[str]		    =   do_lower_case
				__lowerCAmelCase    :		str		    =   strip_accents
				__lowerCAmelCase    :		int		    =   tokenize_chinese_chars
				__lowerCAmelCase    :		Optional[int]		    =   normalizer_class(**_SCREAMING_SNAKE_CASE				)

			__lowerCAmelCase    :		str		    =   do_lower_case

		def 							__lowerCamelCase							(   self ,	_SCREAMING_SNAKE_CASE ,	_SCREAMING_SNAKE_CASE=None				):
			__lowerCAmelCase    :		List[str]		    =   [self.cls_token_id] + token_ids_a + [self.sep_token_id]

			if token_ids_a:
				output += token_ids_a + [self.sep_token_id]

			return output

		def 							__lowerCamelCase							(   self ,	_SCREAMING_SNAKE_CASE ,	_SCREAMING_SNAKE_CASE = None				):
			__lowerCAmelCase    :		Union[str, Any]		    =   [self.sep_token_id]
			__lowerCAmelCase    :		Any		    =   [self.cls_token_id]
			if token_ids_a is None:
				return len(cls + token_ids_a + sep				) * [0]
			return len(cls + token_ids_a + sep				) * [0] + len(token_ids_a + sep				) * [1]







		def 							__lowerCamelCase							(   self ,	_SCREAMING_SNAKE_CASE ,	_SCREAMING_SNAKE_CASE = None				):
			__lowerCAmelCase    :		Optional[int]		    =   self._tokenizer.model.save(_SCREAMING_SNAKE_CASE ,	name=_SCREAMING_SNAKE_CASE				)
			return tuple(_SCREAMING_SNAKE_CASE				)

"""simple docstring"""







import unittest

import numpy as np

from transformers import RobertaConfig, is_flax_available
from transformers.testing_utils import require_flax, slow

from ...test_modeling_flax_common import FlaxModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask


if is_flax_available():
							from transformers.models.roberta.modeling_flax_roberta import (
							    FlaxRobertaForCausalLM,
							    FlaxRobertaForMaskedLM,
							    FlaxRobertaForMultipleChoice,
							    FlaxRobertaForQuestionAnswering,
							    FlaxRobertaForSequenceClassification,
							    FlaxRobertaForTokenClassification,
							    FlaxRobertaModel,
							)




class 				A__ (			unittest.TestCase):

		def __init__(   self ,	_SCREAMING_SNAKE_CASE ,	_SCREAMING_SNAKE_CASE=13 ,	_SCREAMING_SNAKE_CASE=7 ,	_SCREAMING_SNAKE_CASE=True ,	_SCREAMING_SNAKE_CASE=True ,	_SCREAMING_SNAKE_CASE=True ,	_SCREAMING_SNAKE_CASE=True ,	_SCREAMING_SNAKE_CASE=99 ,	_SCREAMING_SNAKE_CASE=32 ,	_SCREAMING_SNAKE_CASE=5 ,	_SCREAMING_SNAKE_CASE=4 ,	_SCREAMING_SNAKE_CASE=37 ,	_SCREAMING_SNAKE_CASE="gelu" ,	_SCREAMING_SNAKE_CASE=0.1 ,	_SCREAMING_SNAKE_CASE=0.1 ,	_SCREAMING_SNAKE_CASE=5_12 ,	_SCREAMING_SNAKE_CASE=16 ,	_SCREAMING_SNAKE_CASE=2 ,	_SCREAMING_SNAKE_CASE=0.02 ,	_SCREAMING_SNAKE_CASE=4 ,	):
			__lowerCAmelCase    :		List[Any]		    =   parent
			__lowerCAmelCase    :		int		    =   batch_size
			__lowerCAmelCase    :		List[str]		    =   seq_length
			__lowerCAmelCase    :		str		    =   is_training
			__lowerCAmelCase    :		Optional[Any]		    =   use_attention_mask
			__lowerCAmelCase    :		Union[str, Any]		    =   use_token_type_ids
			__lowerCAmelCase    :		Dict		    =   use_labels
			__lowerCAmelCase    :		Union[str, Any]		    =   vocab_size
			__lowerCAmelCase    :		List[str]		    =   hidden_size
			__lowerCAmelCase    :		List[str]		    =   num_hidden_layers
			__lowerCAmelCase    :		Optional[int]		    =   num_attention_heads
			__lowerCAmelCase    :		Dict		    =   intermediate_size
			__lowerCAmelCase    :		Dict		    =   hidden_act
			__lowerCAmelCase    :		Optional[int]		    =   hidden_dropout_prob
			__lowerCAmelCase    :		Dict		    =   attention_probs_dropout_prob
			__lowerCAmelCase    :		Union[str, Any]		    =   max_position_embeddings
			__lowerCAmelCase    :		Any		    =   type_vocab_size
			__lowerCAmelCase    :		Tuple		    =   type_sequence_label_size
			__lowerCAmelCase    :		Union[str, Any]		    =   initializer_range
			__lowerCAmelCase    :		int		    =   num_choices

		def 							__lowerCamelCase							(   self				):
			__lowerCAmelCase    :		Union[str, Any]		    =   ids_tensor([self.batch_size, self.seq_length] ,	self.vocab_size				)

			__lowerCAmelCase    :		List[str]		    =   None
			if self.use_attention_mask:
				__lowerCAmelCase    :		str		    =   random_attention_mask([self.batch_size, self.seq_length]				)

			__lowerCAmelCase    :		Dict		    =   None
			if self.use_token_type_ids:
				__lowerCAmelCase    :		Tuple		    =   ids_tensor([self.batch_size, self.seq_length] ,	self.type_vocab_size				)

			__lowerCAmelCase    :		str		    =   RobertaConfig(
			    vocab_size=self.vocab_size ,	hidden_size=self.hidden_size ,	num_hidden_layers=self.num_hidden_layers ,	num_attention_heads=self.num_attention_heads ,	intermediate_size=self.intermediate_size ,	hidden_act=self.hidden_act ,	hidden_dropout_prob=self.hidden_dropout_prob ,	attention_probs_dropout_prob=self.attention_probs_dropout_prob ,	max_position_embeddings=self.max_position_embeddings ,	type_vocab_size=self.type_vocab_size ,	is_decoder=_SCREAMING_SNAKE_CASE ,	initializer_range=self.initializer_range ,	)

			return config, input_ids, token_type_ids, attention_mask

		def 							__lowerCamelCase							(   self				):
			__lowerCAmelCase    :		Dict		    =   self.prepare_config_and_inputs()
			__lowerCAmelCase    ,     __lowerCAmelCase    ,     __lowerCAmelCase    ,     __lowerCAmelCase    :		List[Any]		    =   config_and_inputs
			__lowerCAmelCase    :		List[Any]		    =   {'input_ids': input_ids, 'token_type_ids': token_type_ids, 'attention_mask': attention_mask}
			return config, inputs_dict







		def 							__lowerCamelCase							(   self				):
			__lowerCAmelCase    :		Optional[int]		    =   self.prepare_config_and_inputs()
			__lowerCAmelCase    ,     __lowerCAmelCase    ,     __lowerCAmelCase    ,     __lowerCAmelCase    :		Any		    =   config_and_inputs

			__lowerCAmelCase    :		int		    =   True
			__lowerCAmelCase    :		Any		    =   floats_tensor([self.batch_size, self.seq_length, self.hidden_size]				)
			__lowerCAmelCase    :		int		    =   ids_tensor([self.batch_size, self.seq_length] ,	vocab_size=2				)

			return (
			    config,
			    input_ids,
			    token_type_ids,
			    encoder_hidden_states,
			    encoder_attention_mask,
			)




@require_flax
class 				A__ (			_lowerCamelCase ,  unittest.TestCase):
		A_		: List[Any]            =			True

		A_		: Optional[Any]            =			(
		    (
		        FlaxRobertaModel,
		        FlaxRobertaForCausalLM,
		        FlaxRobertaForMaskedLM,
		        FlaxRobertaForSequenceClassification,
		        FlaxRobertaForTokenClassification,
		        FlaxRobertaForMultipleChoice,
		        FlaxRobertaForQuestionAnswering,
		    )
		    if is_flax_available()
		    else ()
		)

		def 							__lowerCamelCase							(   self				):
			__lowerCAmelCase    :		Tuple		    =   FlaxRobertaModelTester(self				)







		@slow
		def 							__lowerCamelCase							(   self				):
			for model_class_name in self.all_model_classes:
				__lowerCAmelCase    :		str		    =   model_class_name.from_pretrained('roberta-base' ,	from_pt=_SCREAMING_SNAKE_CASE				)
				__lowerCAmelCase    :		Optional[int]		    =   model(np.ones((1, 1)				)				)
				self.assertIsNotNone(_SCREAMING_SNAKE_CASE				)